25 #ifndef __TERRALIB_RP_INTERNAL_MATRIX_H 26 #define __TERRALIB_RP_INTERNAL_MATRIX_H 30 #include "../common/PlatformUtils.h" 31 #include "../core/filesystem/FileSystem.h" 32 #include "../core/encoding/CharEncoding.h" 33 #include "../core/utils/Platform.h" 42 #include <boost/scoped_array.hpp> 43 #include <boost/filesystem.hpp> 53 template<
typename TemplateElementType >
114 bool reset(
unsigned int lines,
unsigned int columns );
126 bool reset(
unsigned int lines,
unsigned int columns,
141 bool reset(
unsigned int lines,
unsigned int columns,
142 MemoryPolicy memoryPolicy,
unsigned char maxMemPercentUsage );
157 bool reset(
unsigned int lines,
unsigned int columns,
158 MemoryPolicy memoryPolicy,
unsigned long int maxTmpFileSize,
159 unsigned char maxMemPercentUsage );
207 const unsigned int& line,
208 const unsigned int& column )
224 const unsigned int& line,
225 const unsigned int& column )
const 245 const unsigned int& line )
262 const unsigned int& line )
const 408 const unsigned int& line )
const;
418 std::string& fullFileName )
const;
421 template<
typename TemplateElementType >
428 template<
typename TemplateElementType >
433 template<
typename TemplateElementType >
439 template<
typename TemplateElementType >
444 remove( m_fullFileName.c_str() );
449 template<
typename TemplateElementType >
461 template<
typename TemplateElementType >
467 template<
typename TemplateElementType >
476 template<
typename TemplateElementType >
482 template<
typename TemplateElementType >
488 template<
typename TemplateElementType >
494 template<
typename TemplateElementType >
496 unsigned int columns )
502 template<
typename TemplateElementType >
510 template<
typename TemplateElementType >
513 unsigned char maxMemPercentUsage )
516 maxMemPercentUsage );
519 template<
typename TemplateElementType >
522 unsigned long int maxTmpFileSize,
523 unsigned char maxMemPercentUsage )
535 if( ( lines > 0 ) && ( columns > 0 ) )
544 const unsigned int lineSizeBytes = (
unsigned int)(
556 TemplateElementType* newLinePtr = 0;
558 for(
unsigned int allLinesPtrsVecIdx = 0 ; allLinesPtrsVecIdx <
580 unsigned int maxRAMLines = 1;
592 const double freeVMem =
MIN( totalPhysMem,
593 ( totalVMem - usedVMem ) );
598 maxRAMLines = (
unsigned int)
MAX( 1.0, linesMem /
599 ((
double)lineSizeBytes) );
604 const unsigned int ramLinesIndexesVecSize =
MIN( maxRAMLines,
610 TemplateElementType* newLinePtr = 0;
612 for(
unsigned int allLinesPtrsVecIdx = 0 ; allLinesPtrsVecIdx <
613 ramLinesIndexesVecSize ; ++allLinesPtrsVecIdx )
644 template<
typename TemplateElementType >
647 TemplateElementType* linePtr = 0;
648 for(
unsigned int memoryblocksHandlerIdx = 0 ; memoryblocksHandlerIdx <
655 delete[] ( linePtr );
673 template<
typename TemplateElementType >
680 template<
typename TemplateElementType >
687 template<
typename TemplateElementType >
693 template<
typename TemplateElementType >
700 "Unable to initiate the matrix object" );
702 unsigned int column = 0;;
703 TemplateElementType
const* inLinePtr = 0;
704 TemplateElementType* outLinePtr = 0;
706 for(
unsigned int line = 0 ; line <
m_totalLines ; ++line )
712 outLinePtr[ column ] = inLinePtr[ column ];
719 template<
typename TemplateElementType >
725 template<
typename TemplateElementType >
731 template<
typename TemplateElementType >
737 template<
typename TemplateElementType >
740 const unsigned long int diskLinesNmb =
m_totalLines - startingLineIdx;
744 const unsigned long int lineSizeBytes = (
unsigned long int)(
747 const unsigned long int maxLinesPerFile = (
unsigned long int )
750 const unsigned long int maxFileSize = (
unsigned long int)
751 ( maxLinesPerFile * lineSizeBytes );
753 const unsigned int openDiskFilesNumber = (
unsigned int)ceil( ((
double)diskLinesNmb)
754 / ((
double)maxLinesPerFile) );
760 unsigned int remainingLinesNmb = diskLinesNmb;
761 unsigned int fileSize = 0;
762 unsigned int fileLinesNumber = 0;
763 unsigned int diskLinesVecIdx = startingLineIdx;
765 for(
unsigned int fileIdx = 0 ; fileIdx < openDiskFilesNumber ;
770 fileSize = maxFileSize;
771 fileLinesNumber = maxLinesPerFile;
773 if( remainingLinesNmb < maxLinesPerFile )
775 fileSize = (
unsigned long int)( lineSizeBytes * remainingLinesNmb );
776 fileLinesNumber = remainingLinesNmb;
779 remainingLinesNmb -= fileLinesNumber;
783 FILE* newFilePtr = 0;
784 std::string newFullFileName;
788 TERP_LOGERR(
"Unable to create temporary disk file" );
797 for(
unsigned int lineIdx = 0; lineIdx < fileLinesNumber ; ++lineIdx )
812 template<
typename TemplateElementType >
843 (
long)( inLineData.m_fileOff ), SEEK_SET ),
848 (
size_t)(
sizeof( TemplateElementType ) *
m_totalColumns ), 1, inLineData.m_filePtr ),
854 (
long)( inLineData.m_fileOff ), SEEK_SET ),
862 inLineData.m_filePtr ),
"File write error" )
876 outLineData.m_filePtr = inLineData.m_filePtr;
877 outLineData.m_fileOff = inLineData.m_fileOff;
879 inLineData.m_filePtr = 0;
880 inLineData.m_fileOff = 0;
893 template<
typename TemplateElementType >
895 FILE** fileptr, std::string& fullFileName )
const 899 bool returnValue =
true;
900 long seekoff = (long)size;
901 const boost::filesystem::path randShortFileNamePath(
902 "TerralibRPMatrix_%%%%-%%%%-%%%%-%%%%" );
903 const unsigned char c =
'\0';
907 randShortFileNamePath).
string());
909 if( fullFileName.empty() )
918 if( (*fileptr) == 0 )
924 if( std::fseek( (*fileptr), seekoff, SEEK_SET ) != 0 )
926 std::fclose( (*fileptr) );
927 std::remove( fullFileName.c_str() );
932 if( 1 != std::fwrite( &c, 1, 1, (*fileptr) ) )
934 std::fclose( (*fileptr) );
935 std::remove( fullFileName.c_str() );
945 std::fclose( (*fileptr) );
948 std::remove( fullFileName.c_str() );
959 randShortFileNamePath).
string());
961 if( fullFileName.empty() )
970 if( (*fileptr) == 0 )
976 if( std::fseek( (*fileptr), seekoff, SEEK_SET ) != 0 )
978 std::fclose( (*fileptr) );
979 std::remove( fullFileName.c_str() );
984 if( 1 != std::fwrite( &c, 1, 1, (*fileptr) ) )
986 std::fclose( (*fileptr) );
987 std::remove( fullFileName.c_str() );
997 std::fclose( (*fileptr) );
1000 std::remove( fullFileName.c_str() );
1001 returnValue =
false;
1010 #endif //__TERRALIB_RP_INTERNAL_MATRIX_H TECOMMONEXPORT unsigned long long int GetTotalVirtualMemory()
Returns the amount of total virtual memory (bytes) that can be claimed by the current process (physic...
void clear()
Clear all allocated resources and go back to the initial default parameters.
unsigned int m_fileOff
The offset within the file.
bool isEmpty() const
Empty Matrix verification.
static std::string uniquePath(const std::string &format="%%%%-%%%%-%%%%-%%%%")
Retrives an unique path generated by a given format.
unsigned char getMaxMemPercentUsage() const
Returns the max amount of free memory to use when necessary.
FILE * m_filePtr
The file pointer.
unsigned char m_maxMemPercentUsage
the max amount of free memory to use when necessary.
unsigned long int getMaxTmpFileSize() const
Returns the current maximum temporary disk file size.
std::vector< DiskLineInfo > m_diskLinesInfoVec
Info of all lines, used when a line is on disk.
#define MIN(a, b)
Macro that returns min between two values.
#define TERP_LOGERR(message)
Logs an error message.
TemplateElementType & operator()(const unsigned int &line, const unsigned int &column)
Operator () overload.
bool allocateDiskLines(unsigned int startingLineIdx)
Allocate disk lines.
bool createNewDiskFile(unsigned long int size, FILE **fileptr, std::string &fullFileName) const
Create a new disk file.
#define MAX(a, b)
Macro that returns max between two values.
void init()
Reset the internal variables to the initial state.
TECOMMONEXPORT unsigned long long int GetTotalPhysicalMemory()
Returns the amount of total physical memory (bytes).
MemoryPolicy m_memoryPolicy
The used memory policy (default:RAMMemPol).
unsigned long int m_maxTmpFileSize
Max bytes per temp file (for swapped tiles only, default: 2GB).
TECOREEXPORT std::string GetUserDirectory()
Returns the system user home dir path.
std::vector< unsigned int > m_ramLinesIndexesVec
The indexes of all lines loaded into RAM.
MemoryPolicy
Memory polycy.
std::vector< TemplateElementType *> m_allLinesPtrsVec
A vector with pointers to all lines.
Openend disk files info node.
unsigned int m_totalColumns
The total columns number (default:0).
const Matrix< TemplateElementType > & operator=(const Matrix< TemplateElementType > &external)
Operator = overload.
TECOMMONEXPORT unsigned long long int GetUsedVirtualMemory()
Returns the amount of used virtual memory (bytes) for the current process (physical + swapped)...
TemplateElementType * getScanLine(const unsigned int &line) const
Returns a pointer to the required line.
MemoryPolicy getMemPolicy() const
Returns the current memory policy.
static std::string fromUTF8(const std::string &src)
Convert a string in UTF-8 to the current locale encoding.
const TemplateElementType & operator()(const unsigned int &line, const unsigned int &column) const
Operator () overload.
std::vector< OpenDiskFileHandler > m_openDiskFilesHandler
A vector with open disk files handler.
void reset()
Reset (clear) the active instance data.
unsigned int getLinesNumber() const
The number of current matrix lines.
#define TERP_LOG_AND_RETURN_FALSE(message)
Logs a warning message will and return false.
std::vector< TemplateElementType *> m_memoryblocksHandler
A generic template matrix.
TemplateElementType const * operator[](const unsigned int &line) const
Operator [] overload.
TemplateElementType ElementTypeT
Public matrix element type definition.
FILE * m_filePtr
The file pointer.
TemplateElementType * m_currentSwapTilePtr
A pointer to the current swap tile.
unsigned int m_nextSwapLineRamLinesIndexesVecIdx
The index inside ramLinesIndexesVec_ of the next RAM line index that will be swapped to disk when a d...
#define TERP_DEBUG_TRUE_OR_THROW(value, message)
Checks if value is true and throws an exception if not.
std::string m_fullFileName
The full created file name.
unsigned int getColumnsNumber() const
The number of current matrix columns.
unsigned int m_totalLines
The total lines number (default:0).
TemplateElementType * operator[](const unsigned int &line)
Operator [] overload.
#define TERP_TRUE_OR_THROW(value, message)
Checks if value is true and throws an exception if not.
boost::scoped_array< TemplateElementType > m_swapMemoryBlockHandler
An auxiliar line used when swapping data to/from disk.