25 #ifndef __TERRALIB_RP_INTERNAL_MATRIX_H 26 #define __TERRALIB_RP_INTERNAL_MATRIX_H 30 #include "../common/PlatformUtils.h" 31 #include "../core/filesystem/FileSystem.h" 32 #include "../core/encoding/CharEncoding.h" 33 #include "../core/utils/Platform.h" 43 #include <boost/scoped_array.hpp> 44 #include <boost/filesystem.hpp> 54 template<
typename TemplateElementType >
115 bool reset(
unsigned int lines,
unsigned int columns );
128 bool reset(
unsigned int lines,
unsigned int columns,
144 bool reset(
unsigned int lines,
unsigned int columns,
145 MemoryPolicy memoryPolicy,
unsigned char maxMemPercentUsage );
160 bool reset(
unsigned int lines,
unsigned int columns,
161 MemoryPolicy memoryPolicy,
unsigned long int maxTmpFileSize,
162 unsigned char maxMemPercentUsage );
173 unsigned int columns,
unsigned int maxRAMLines );
184 unsigned int columns,
unsigned long int maxTmpFileSize,
185 unsigned int maxRAMLines );
228 const unsigned int& line,
229 const unsigned int& column )
245 const unsigned int& line,
246 const unsigned int& column )
const 266 const unsigned int& line )
283 const unsigned int& line )
const 427 const unsigned int& line )
const;
437 std::string& fullFileName )
const;
440 template<
typename TemplateElementType >
447 template<
typename TemplateElementType >
452 template<
typename TemplateElementType >
458 template<
typename TemplateElementType >
463 std::fclose( m_filePtr );
464 remove( m_fullFileName.c_str() );
468 template<
typename TemplateElementType >
480 template<
typename TemplateElementType >
486 template<
typename TemplateElementType >
495 template<
typename TemplateElementType >
501 template<
typename TemplateElementType >
507 template<
typename TemplateElementType >
514 template<
typename TemplateElementType >
516 unsigned int columns )
521 template<
typename TemplateElementType >
528 template<
typename TemplateElementType >
531 unsigned char maxMemPercentUsage )
534 maxMemPercentUsage );
537 template<
typename TemplateElementType >
540 unsigned long int maxTmpFileSize,
541 unsigned char maxMemPercentUsage )
545 unsigned int maxRAMLines = std::numeric_limits< unsigned int >::max();
551 const unsigned int lineSizeBytes = (
unsigned int)(
552 sizeof( TemplateElementType ) * columns );
560 const double freeVMem = std::min( totalPhysMem,
561 ( totalVMem - usedVMem ) );
563 const double linesMem = ( ((double)maxMemPercentUsage) / 100.0 ) *
566 maxRAMLines = (
unsigned int)std::max( 1.0, linesMem /
567 ((
double)lineSizeBytes) );
570 return reset( memoryPolicy, lines, columns, maxTmpFileSize, maxRAMLines );
573 template<
typename TemplateElementType >
575 unsigned int lines,
unsigned int columns,
unsigned int maxRAMLines)
581 template<
typename TemplateElementType >
583 unsigned int lines,
unsigned int columns,
unsigned long int maxTmpFileSize,
584 unsigned int maxRAMLines )
608 if( ( lines > 0 ) && ( columns > 0 ) )
627 TemplateElementType* newLinePtr = 0;
629 for(
unsigned int allLinesPtrsVecIdx = 0 ; allLinesPtrsVecIdx <
651 const unsigned int ramLinesIndexesVecSize = std::min(
m_maxRAMLines,
657 TemplateElementType* newLinePtr = 0;
659 for(
unsigned int allLinesPtrsVecIdx = 0 ; allLinesPtrsVecIdx <
660 ramLinesIndexesVecSize ; ++allLinesPtrsVecIdx )
673 const unsigned long int diskLinesNmb =
m_totalLines - ramLinesIndexesVecSize;
677 const unsigned long int lineSizeBytes = (
unsigned long int)(
680 const unsigned long int maxLinesPerFile = (
unsigned long int )
683 const unsigned long int maxFileSize = (
unsigned long int)
684 ( maxLinesPerFile * lineSizeBytes );
686 const unsigned int openDiskFilesNumber = (
unsigned int)ceil( ((
double)diskLinesNmb)
687 / ((
double)maxLinesPerFile) );
694 unsigned int remainingLinesNmb = diskLinesNmb;
695 unsigned int fileSize = 0;
696 unsigned int fileLinesNumber = 0;
697 unsigned int diskLinesVecIdx = ramLinesIndexesVecSize;
699 for(
unsigned int fileIdx = 0 ; fileIdx < openDiskFilesNumber ;
704 fileSize = maxFileSize;
705 fileLinesNumber = maxLinesPerFile;
707 if( remainingLinesNmb < maxLinesPerFile )
709 fileSize = (
unsigned long int)( lineSizeBytes * remainingLinesNmb );
710 fileLinesNumber = remainingLinesNmb;
713 remainingLinesNmb -= fileLinesNumber;
717 std::FILE* newFilePtr = 0;
718 std::string newFullFileName;
731 for(
unsigned int fileLineIdx = 0; fileLineIdx < fileLinesNumber ; ++fileLineIdx )
755 template<
typename TemplateElementType >
758 TemplateElementType* linePtr = 0;
759 for(
unsigned int memoryblocksHandlerIdx = 0 ; memoryblocksHandlerIdx <
766 delete[] ( linePtr );
790 template<
typename TemplateElementType >
797 template<
typename TemplateElementType >
804 template<
typename TemplateElementType >
810 template<
typename TemplateElementType >
817 "Unable to initiate the matrix object" );
819 unsigned int column = 0;;
820 TemplateElementType
const* inLinePtr = 0;
821 TemplateElementType* outLinePtr = 0;
823 for(
unsigned int line = 0 ; line <
m_totalLines ; ++line )
829 outLinePtr[ column ] = inLinePtr[ column ];
836 template<
typename TemplateElementType >
842 template<
typename TemplateElementType >
848 template<
typename TemplateElementType >
854 template<
typename TemplateElementType >
885 (
long)( inLineData.m_fileOff ), SEEK_SET ),
890 (
size_t)(
sizeof( TemplateElementType ) *
m_totalColumns ), 1, inLineData.m_filePtr ),
896 (
long)( inLineData.m_fileOff ), SEEK_SET ),
904 inLineData.m_filePtr ),
"File write error" )
918 outLineData.m_filePtr = inLineData.m_filePtr;
919 outLineData.m_fileOff = inLineData.m_fileOff;
921 inLineData.m_filePtr = 0;
922 inLineData.m_fileOff = 0;
935 template<
typename TemplateElementType >
937 FILE** fileptr, std::string& fullFileName )
const 941 bool returnValue =
true;
942 long seekoff = (long)size;
943 const boost::filesystem::path randShortFileNamePath(
944 "TerralibRPMatrix_%%%%-%%%%-%%%%-%%%%" );
945 const unsigned char c =
'\0';
949 randShortFileNamePath).
string());
951 if( fullFileName.empty() )
960 if( (*fileptr) == 0 )
966 if( std::fseek( (*fileptr), seekoff, SEEK_SET ) != 0 )
968 std::fclose( (*fileptr) );
969 std::remove( fullFileName.c_str() );
974 if( 1 != std::fwrite( &c, 1, 1, (*fileptr) ) )
976 std::fclose( (*fileptr) );
977 std::remove( fullFileName.c_str() );
987 std::fclose( (*fileptr) );
990 std::remove( fullFileName.c_str() );
1001 randShortFileNamePath).
string());
1003 if( fullFileName.empty() )
1005 returnValue =
false;
1012 if( (*fileptr) == 0 )
1014 returnValue =
false;
1018 if( std::fseek( (*fileptr), seekoff, SEEK_SET ) != 0 )
1020 std::fclose( (*fileptr) );
1021 std::remove( fullFileName.c_str() );
1022 returnValue =
false;
1026 if( 1 != std::fwrite( &c, 1, 1, (*fileptr) ) )
1028 std::fclose( (*fileptr) );
1029 std::remove( fullFileName.c_str() );
1030 returnValue =
false;
1039 std::fclose( (*fileptr) );
1042 std::remove( fullFileName.c_str() );
1043 returnValue =
false;
1052 #endif //__TERRALIB_RP_INTERNAL_MATRIX_H TECOMMONEXPORT unsigned long long int GetTotalVirtualMemory()
Returns the amount of total virtual memory (bytes) that can be claimed by the current process (physic...
void clear()
Clear all allocated resources but keep the current parameters.
unsigned int m_fileOff
The offset within the file.
bool isEmpty() const
Empty Matrix verification.
static std::string uniquePath(const std::string &format="%%%%-%%%%-%%%%-%%%%")
Retrives an unique path generated by a given format.
unsigned long int getMaxTmpFileSize() const
Returns the current maximum temporary disk file size.
std::vector< DiskLineInfo > m_diskLinesInfoVec
Info of all lines, used when a line is on disk.
TemplateElementType & operator()(const unsigned int &line, const unsigned int &column)
Operator () overload.
bool createNewDiskFile(unsigned long int size, FILE **fileptr, std::string &fullFileName) const
Create a new disk file.
void init()
Reset the internal variables to the default state.
TECOMMONEXPORT unsigned long long int GetTotalPhysicalMemory()
Returns the amount of total physical memory (bytes).
MemoryPolicy m_memoryPolicy
The used memory policy (default:RAMMemPol).
unsigned long int m_maxTmpFileSize
Max bytes per temp file (for swapped tiles only, default: 2GB).
TECOREEXPORT std::string GetUserDirectory()
Returns the system user home dir path.
std::vector< unsigned int > m_ramLinesIndexesVec
The indexes of all lines loaded into RAM.
MemoryPolicy
Memory polycy.
std::vector< TemplateElementType *> m_allLinesPtrsVec
A vector with pointers to all lines.
Openend disk files info node.
unsigned int m_totalColumns
The total columns number (default:0).
const Matrix< TemplateElementType > & operator=(const Matrix< TemplateElementType > &external)
Operator = overload.
TECOMMONEXPORT unsigned long long int GetUsedVirtualMemory()
Returns the amount of used virtual memory (bytes) for the current process (physical + swapped)...
TemplateElementType * getScanLine(const unsigned int &line) const
Returns a pointer to the required line.
MemoryPolicy getMemPolicy() const
Returns the current memory policy.
static std::string fromUTF8(const std::string &src)
Convert a string in UTF-8 to the current locale encoding.
const TemplateElementType & operator()(const unsigned int &line, const unsigned int &column) const
Operator () overload.
std::vector< OpenDiskFileHandler > m_openDiskFilesHandler
A vector with open disk files handler.
void reset()
Reset (clear) the active instance data.
unsigned int getLinesNumber() const
The number of current matrix lines.
#define TERP_LOG_AND_RETURN_FALSE(message)
Logs a warning message will and return false.
std::vector< TemplateElementType *> m_memoryblocksHandler
A generic template matrix.
std::FILE * m_filePtr
The file pointer.
TemplateElementType const * operator[](const unsigned int &line) const
Operator [] overload.
TemplateElementType ElementTypeT
Public matrix element type definition.
TemplateElementType * m_currentSwapTilePtr
A pointer to the current swap tile.
unsigned int m_nextSwapLineRamLinesIndexesVecIdx
The index inside ramLinesIndexesVec_ of the next RAM line index that will be swapped to disk when a d...
std::FILE * m_filePtr
The file pointer.
#define TERP_DEBUG_TRUE_OR_THROW(value, message)
Checks if value is true and throws an exception if not.
std::string m_fullFileName
The full created file name.
unsigned int getColumnsNumber() const
The number of current matrix columns.
unsigned int m_totalLines
The total lines number (default:0).
unsigned int getMaxRAMLines() const
Returns the allowed maximum amount of RAM lines.
TemplateElementType * operator[](const unsigned int &line)
Operator [] overload.
unsigned int m_maxRAMLines
the max amount of RAM lines.
#define TERP_TRUE_OR_THROW(value, message)
Checks if value is true and throws an exception if not.
boost::scoped_array< TemplateElementType > m_swapMemoryBlockHandler
An auxiliar line used when swapping data to/from disk.