27 #include "../common/Translator.h"
28 #include "../common/PlatformUtils.h"
29 #include "../geometry/Envelope.h"
30 #include "../raster/Utils.h"
31 #include "../raster/BlockUtils.h"
36 #ifndef TLINTERNAL_EXPANSIBLERASTER_MAXDISKFILESSIZE
37 #define TLINTERNAL_EXPANSIBLERASTER_MAXDISKFILESSIZE 2147483648ul
50 : te::rst::
Raster( grid, p )
56 const std::vector<te::rst::BandProperty*> bandsProperties )
61 unsigned int maxBlockSizeBytes = 0;
62 std::vector< unsigned int> numbersOfBlocksX;
63 std::vector< unsigned int> numbersOfBlocksY;
64 std::vector< unsigned int> blocksSizesBytes;
67 for(
unsigned int bandIdx = 0 ; bandIdx < bandsProperties.size() ; ++bandIdx )
70 bandsProperties[ bandIdx ]->m_nblocksx = bandsProperties[ 0 ]->m_nblocksx;
71 bandsProperties[ bandIdx ]->m_nblocksy = bandsProperties[ 0 ]->m_nblocksy;
72 bandsProperties[ bandIdx ]->m_blkw = bandsProperties[ 0 ]->m_blkw;
73 bandsProperties[ bandIdx ]->m_blkh = bandsProperties[ 0 ]->m_blkh;
74 bandsProperties[ bandIdx ]->m_nblocksx = (int)std::ceil(
76 ((
double)bandsProperties[ bandIdx ]->m_blkw) );
77 bandsProperties[ bandIdx ]->m_nblocksy = (int)std::ceil(
79 ((
double)bandsProperties[ bandIdx ]->m_blkh) );
81 const unsigned int blockSizeBytes = (
unsigned int)(
82 bandsProperties[ bandIdx ]->m_blkw *
83 bandsProperties[ bandIdx ]->m_blkh *
86 if( maxBlockSizeBytes < blockSizeBytes )
87 maxBlockSizeBytes = blockSizeBytes;
89 numbersOfBlocksX.push_back( (
unsigned int)bandsProperties[ bandIdx ]->m_nblocksx );
90 numbersOfBlocksY.push_back( (
unsigned int)bandsProperties[ bandIdx ]->m_nblocksy );
91 blocksSizesBytes.push_back( blockSizeBytes );
98 const double freeVMem = ( ((double)maxMemPercentUsed) / 100.0 ) *
99 std::min( totalPhysMem, ( totalVMem - usedVMem ) );
100 const unsigned int maxNumberOfBlocks = (
unsigned int)
101 std::max( 1.0, std::ceil( freeVMem / ((
double)maxBlockSizeBytes) ) );
104 numbersOfBlocksY, blocksSizesBytes,
106 throw Exception(
TE_TR(
"Cannot initialize the blocks menager") );
108 for(
unsigned int bandsIdx = 0 ; bandsIdx < bandsProperties.size() ;
112 *(bandsProperties[ bandsIdx ]) , bandsIdx ) );
113 delete ( bandsProperties[ bandsIdx ] );
118 const std::vector<te::rst::BandProperty*> bandsProperties,
119 const unsigned int maxNumberOfRAMBlocks )
124 std::vector< unsigned int> numbersOfBlocksX;
125 std::vector< unsigned int> numbersOfBlocksY;
126 std::vector< unsigned int> blocksSizesBytes;
129 for(
unsigned int bandIdx = 0 ; bandIdx < bandsProperties.size() ; ++bandIdx )
132 bandsProperties[ bandIdx ]->m_nblocksx = bandsProperties[ 0 ]->m_nblocksx;
133 bandsProperties[ bandIdx ]->m_nblocksy = bandsProperties[ 0 ]->m_nblocksy;
134 bandsProperties[ bandIdx ]->m_blkw = bandsProperties[ 0 ]->m_blkw;
135 bandsProperties[ bandIdx ]->m_blkh = bandsProperties[ 0 ]->m_blkh;
136 bandsProperties[ bandIdx ]->m_nblocksx = (int)std::ceil(
138 ((
double)bandsProperties[ bandIdx ]->m_blkw) );
139 bandsProperties[ bandIdx ]->m_nblocksy = (int)std::ceil(
141 ((
double)bandsProperties[ bandIdx ]->m_blkh) );
143 const unsigned int blockSizeBytes = (
unsigned int)(
144 bandsProperties[ bandIdx ]->m_blkw *
145 bandsProperties[ bandIdx ]->m_blkh *
148 numbersOfBlocksX.push_back( (
unsigned int)bandsProperties[ bandIdx ]->m_nblocksx );
149 numbersOfBlocksY.push_back( (
unsigned int)bandsProperties[ bandIdx ]->m_nblocksy );
150 blocksSizesBytes.push_back( blockSizeBytes );
155 numbersOfBlocksY, blocksSizesBytes,
157 throw Exception(
TE_TR(
"Cannot initialize the blocks menager") );
159 for(
unsigned int bandsIdx = 0 ; bandsIdx < bandsProperties.size() ;
163 *(bandsProperties[ bandsIdx ]) , bandsIdx ) );
164 delete ( bandsProperties[ bandsIdx ] );
180 return std::map<std::string, std::string>();
185 std::vector<te::rst::BandProperty*> bandsProperties;
186 for(
unsigned int bandsIdx = 0 ; bandsIdx < m_bands.size() ; ++bandsIdx )
189 *(m_bands[ bandsIdx ]->getProperty() ) ) );
193 m_blocksManager.getMaxNumberOfRAMBlocks() );
198 if( m_bands.empty() )
return false;
202 const unsigned int blockExpansionSize = (
unsigned int)std::ceil(
203 ((
double)number) / ((
double)m_bands[ 0 ]->getProperty()->m_blkh) );
205 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > addedBlocksCoords;
207 for(
unsigned int bandsIdx = 0 ; bandsIdx < m_bands.size() ; ++bandsIdx )
209 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > bandAddedBlocksCoords;
210 if( ! m_blocksManager.addTopBlocks( blockExpansionSize, bandsIdx,
211 bandAddedBlocksCoords ) )
214 addedBlocksCoords.insert( addedBlocksCoords.end(), bandAddedBlocksCoords.begin(),
215 bandAddedBlocksCoords.end() );
217 m_bands[ bandsIdx ]->getProperty()->m_nblocksy += blockExpansionSize;
220 dummyFillBlocks( addedBlocksCoords );
222 const unsigned int newLinesNumber =
223 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkh) *
224 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_nblocksy );
225 const unsigned int newColsNumber =
226 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkw) *
227 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_nblocksx );
229 const double addedLinesNumber = (double)( blockExpansionSize *
230 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkh) );
233 m_grid->getExtent()->getLowerLeftX(),
234 m_grid->getExtent()->getUpperRightY() + ( addedLinesNumber *
235 m_grid->getResolutionY() ) );
238 newLinesNumber, m_grid->getResolutionX(), m_grid->getResolutionY(),
239 &newULC, m_grid->getSRID() ) );
249 if( m_bands.empty() )
return false;
253 const unsigned int currExtraLines =
254 ( (
unsigned int)( m_bands[ 0 ]->getProperty()->m_blkh *
255 m_bands[ 0 ]->getProperty()->m_nblocksy ) ) -
256 m_grid->getNumberOfRows();
259 m_grid->getExtent()->getLowerLeftX(),
260 m_grid->getExtent()->getUpperRightY() );
262 if( currExtraLines < number )
264 const unsigned int blockExpansionSize = (
unsigned int)std::ceil(
265 ((
double)number) / ((
double)m_bands[ 0 ]->getProperty()->m_blkh) );
267 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > addedBlocksCoords;
269 for(
unsigned int bandsIdx = 0 ; bandsIdx < m_bands.size() ; ++bandsIdx )
271 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > bandAddedBlocksCoords;
272 if( ! m_blocksManager.addBottomBlocks( blockExpansionSize, bandsIdx,
273 bandAddedBlocksCoords ) )
276 addedBlocksCoords.insert( addedBlocksCoords.end(), bandAddedBlocksCoords.begin(),
277 bandAddedBlocksCoords.end() );
279 m_bands[ bandsIdx ]->getProperty()->m_nblocksy += blockExpansionSize;
282 dummyFillBlocks( addedBlocksCoords );
284 const unsigned int newLinesNumber =
285 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkh) *
286 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_nblocksy );
287 const unsigned int newColsNumber =
288 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkw) *
289 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_nblocksx );
292 newLinesNumber, m_grid->getResolutionX(), m_grid->getResolutionY(),
293 &uLC, m_grid->getSRID() ) );
300 m_grid->getNumberOfColumns(),
301 m_grid->getNumberOfRows() + number,
302 m_grid->getResolutionX(), m_grid->getResolutionY(),
303 &uLC, m_grid->getSRID() ) );
314 if( m_bands.empty() )
return false;
318 const unsigned int oldColsNumber =
319 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkw) *
320 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_nblocksx );
322 const unsigned int blockExpansionSize = (
unsigned int)std::ceil(
323 ((
double)number) / ((
double)m_bands[ 0 ]->getProperty()->m_blkw) );
325 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > addedBlocksCoords;
327 for(
unsigned int bandsIdx = 0 ; bandsIdx < m_bands.size() ; ++bandsIdx )
329 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > bandAddedBlocksCoords;
330 if( ! m_blocksManager.addLeftBlocks( blockExpansionSize, bandsIdx,
331 bandAddedBlocksCoords ) )
334 addedBlocksCoords.insert( addedBlocksCoords.end(), bandAddedBlocksCoords.begin(),
335 bandAddedBlocksCoords.end() );
337 m_bands[ bandsIdx ]->getProperty()->m_nblocksx += blockExpansionSize;
340 dummyFillBlocks( addedBlocksCoords );
342 const unsigned int newLinesNumber =
343 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkh) *
344 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_nblocksy );
345 const unsigned int newColsNumber =
346 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkw) *
347 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_nblocksx );
349 const double addedColsNumber = (double)( newColsNumber - oldColsNumber );
352 m_grid->getExtent()->getLowerLeftX() - ( addedColsNumber *
353 m_grid->getResolutionX() ),
354 m_grid->getExtent()->getUpperRightY() );
357 newLinesNumber, m_grid->getResolutionX(), m_grid->getResolutionY(),
358 &newULC, m_grid->getSRID() ) );
368 if( m_bands.empty() )
return false;
372 const unsigned int currExtraCols =
373 ( (
unsigned int)( m_bands[ 0 ]->getProperty()->m_blkw *
374 m_bands[ 0 ]->getProperty()->m_nblocksx ) ) -
375 m_grid->getNumberOfColumns();
378 m_grid->getExtent()->getLowerLeftX(),
379 m_grid->getExtent()->getUpperRightY() );
381 if( currExtraCols < number )
383 const unsigned int blockExpansionSize = (
unsigned int)std::ceil(
384 ((
double)number) / ((
double)m_bands[ 0 ]->getProperty()->m_blkw) );
386 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > addedBlocksCoords;
388 for(
unsigned int bandsIdx = 0 ; bandsIdx < m_bands.size() ; ++bandsIdx )
390 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > bandAddedBlocksCoords;
391 if( ! m_blocksManager.addRightBlocks( blockExpansionSize, bandsIdx,
392 bandAddedBlocksCoords ) )
395 addedBlocksCoords.insert( addedBlocksCoords.end(), bandAddedBlocksCoords.begin(),
396 bandAddedBlocksCoords.end() );
398 m_bands[ bandsIdx ]->getProperty()->m_nblocksx += blockExpansionSize;
401 dummyFillBlocks( addedBlocksCoords );
403 const unsigned int newLinesNumber =
404 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkh) *
405 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_nblocksy );
406 const unsigned int newColsNumber =
407 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkw) *
408 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_nblocksx );
411 newLinesNumber, m_grid->getResolutionX(), m_grid->getResolutionY(),
412 &uLC, m_grid->getSRID() ) );
419 m_grid->getNumberOfColumns() + number,
420 m_grid->getNumberOfRows(),
421 m_grid->getResolutionX(), m_grid->getResolutionY(),
422 &uLC, m_grid->getSRID() ) );
433 if( m_bands.empty() )
return false;
437 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > addedBlocksCoords;
439 if( ! m_blocksManager.addTopBands( number, addedBlocksCoords ) )
442 m_bands.insert( m_bands.begin(), number, 0 );
443 for(
unsigned int bIdx = 0 ; bIdx < m_bands.size() ; ++bIdx )
445 if( m_bands[ bIdx ] )
450 delete( oldBandPtr );
455 *( m_bands[ number ]->getProperty() ), bIdx );
459 dummyFillBlocks( addedBlocksCoords );
467 if( m_bands.empty() )
return false;
471 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > addedBlocksCoords;
473 if( ! m_blocksManager.addBottomBands( number, addedBlocksCoords ) )
476 unsigned int lastBandIdx = (
unsigned int)m_bands.size() - 1;
477 m_bands.insert( m_bands.end(), number, 0 );
478 for(
unsigned int bIdx = lastBandIdx + 1 ; bIdx < m_bands.size() ; ++bIdx )
481 *( m_bands[ lastBandIdx ]->getProperty() ), bIdx );
484 dummyFillBlocks( addedBlocksCoords );
492 if( m_bands.size() > 0 )
494 for(
unsigned int bandsIdx = 0 ; bandsIdx < m_bands.size() ; ++bandsIdx )
495 delete (m_bands[ bandsIdx ]);
499 m_blocksManager.free();
504 for(
unsigned int bandsIdx = 0 ; bandsIdx < m_bands.size() ;
520 unsigned int blockXIdx = 0;
521 unsigned int blockYIdx = 0;
525 for( blockXIdx = 0 ; blockXIdx < nBlocksX ; ++blockXIdx )
527 for( blockYIdx = 0 ; blockYIdx < nBlocksY ; ++blockYIdx )
529 blockPtr = m_blocksManager.getBlockPointer( bandsIdx, blockXIdx,
533 for( eIdx = 0 ; eIdx < elementsNumber ; ++eIdx )
534 sb( eIdx, blockPtr, noDataValuePtr );
541 const std::vector< ExpansibleBandBlocksManager::BlockIndex3D >& blocksCoords )
543 for(
unsigned int coodsIdx = 0 ; coodsIdx < blocksCoords.size() ;
558 void* blockPtr = m_blocksManager.getBlockPointer( block3DIdx.
m_dim0Index,
561 const int elementsNumber = band.getProperty()->m_blkh *
562 band.getProperty()->m_blkw;
564 double* noDataValuePtr = &( band.getProperty()->m_noDataValue );
566 for(
int eIdx = 0 ; eIdx < elementsNumber ; ++eIdx )
567 sb( eIdx, blockPtr, noDataValuePtr );
A raster (stored in memory and eventually swapped to disk) where it is possible to dynamically add li...
unsigned int getNumberOfRows() const
Returns the grid number of rows.
bool addTopLines(const unsigned int number)
New lines will be added at the top of the raster (before the first line).
std::vector< ExpansibleBand * > m_bands
Internal raster bands.
A raster class for memory.
A raster band description.
int m_nblocksx
The number of blocks in x.
void dummyFillBlocks(const std::vector< ExpansibleBandBlocksManager::BlockIndex3D > &blocksCoords)
Fill the required blocks with dummy values.
int m_nblocksy
The number of blocks in y.
void dummyFillAllBlocks()
Fill all blocks with dummy values.
bool initialize(const unsigned int maxNumberRAMBlocks, const std::vector< unsigned int > &numbersOfBlocksX, const std::vector< unsigned int > &numbersOfBlocksY, const std::vector< unsigned int > &blocksSizesBytes, const unsigned long int maxDiskFilesSize)
Initialize this instance to an initial state.
An utility struct for representing 2D coordinates.
int m_type
The data type of the elements in the band.
double m_noDataValue
Value to indicate elements where there is no data, default is std::numeric_limits::max().
#define TE_TR(message)
It marks a string in order to get translated.
An exception class for the TerraLib In-Memory Data Access driver.
ExpansibleBandBlocksManager m_blocksManager
Internal blocks manager.
void open(const std::map< std::string, std::string > &rinfo, te::common::AccessPolicy p=te::common::RAccess)
Opens a raster.
AccessPolicy
Supported data access policies (can be used as bitfield).
void free()
Free all allocated internal resources and go back to the initial state.
TECOMMONEXPORT unsigned long int GetUsedVirtualMemory()
Returns the amount of used virtual memory (bytes) for the current process (physical + swapped)...
bool addRightColumns(const unsigned int number)
New columns will be added at the right of the raster (after the last column).
std::map< std::string, std::string > getInfo() const
It returns additional information about the raster.
BandProperty * getProperty()
Returns the band property.
CoordDataType m_dim0Index
Block Z (band) index.
int m_blkw
Block width (pixels).
void(* SetBufferValueFPtr)(int index, void *buffer, const double *value)
The type of function used to extract data from a buffer.
bool addBottomBands(const unsigned int number)
New bands will be added at the bottom of the raster (after de the last band).
unsigned int getNumberOfColumns() const
Returns the grid number of columns.
CoordDataType m_dim2Index
Block X index.
bool addBottomLines(const unsigned int number)
New lines will be added at the bottom of the raster (after de the last line).
A base class for values that can be retrieved from the data access module.
TERASTEREXPORT int GetPixelSize(int datatype)
Returns the byte size of a given datatype.
A raster (stored in memory and eventually swapped to disk) where it is possible to dynamically add li...
CoordDataType m_dim1Index
Block Y index.
te::dt::AbstractData * clone() const
It returns a clone of this object.
TECOMMONEXPORT unsigned long int GetTotalPhysicalMemory()
Returns the amount of total physical memory (bytes).
int m_blkh
Block height (pixels).
void(* GetBufferValueFPtr)(int index, void *buffer, double *value)
The type of function used to extract data from a buffer.
A rectified grid is the spatial support for raster data.
bool addLeftColumns(const unsigned int number)
New columns will be added at the left of the raster (before the first column).
bool addTopBands(const unsigned int number)
New bands will be added at the top of the raster (before the first band).
#define TLINTERNAL_EXPANSIBLERASTER_MAXDISKFILESSIZE
TERASTEREXPORT void SetBlockFunctions(GetBufferValueFPtr *gb, GetBufferValueFPtr *gbi, SetBufferValueFPtr *sb, SetBufferValueFPtr *sbi, int type)
Sets the pointers to functions that helps to extract a double or complex value from a specific buffer...
TECOMMONEXPORT unsigned long int GetTotalVirtualMemory()
Returns the amount of total virtual memory (bytes) that can be claimed by the current process (physic...