27 #include "../common/Translator.h"
28 #include "../common/PlatformUtils.h"
29 #include "../geometry/Envelope.h"
30 #include "../raster/Utils.h"
31 #include "../raster/BlockUtils.h"
36 #ifndef TLINTERNAL_EXPANSIBLERASTER_MAXDISKFILESSIZE
37 #define TLINTERNAL_EXPANSIBLERASTER_MAXDISKFILESSIZE 2147483648ul
50 : te::rst::
Raster( grid, p )
56 const std::vector<te::rst::BandProperty*> bandsProperties )
61 unsigned int maxBlockSizeBytes = 0;
62 std::vector< unsigned int> numbersOfBlocksX;
63 std::vector< unsigned int> numbersOfBlocksY;
64 std::vector< unsigned int> blocksSizesBytes;
67 for(
unsigned int bandIdx = 0 ; bandIdx < bandsProperties.size() ; ++bandIdx )
70 bandsProperties[ bandIdx ]->m_nblocksx = bandsProperties[ 0 ]->m_nblocksx;
71 bandsProperties[ bandIdx ]->m_nblocksy = bandsProperties[ 0 ]->m_nblocksy;
72 bandsProperties[ bandIdx ]->m_blkw = bandsProperties[ 0 ]->m_blkw;
73 bandsProperties[ bandIdx ]->m_blkh = bandsProperties[ 0 ]->m_blkh;
74 bandsProperties[ bandIdx ]->m_nblocksx = (int)std::ceil(
76 ((
double)bandsProperties[ bandIdx ]->m_blkw) );
77 bandsProperties[ bandIdx ]->m_nblocksy = (int)std::ceil(
79 ((
double)bandsProperties[ bandIdx ]->m_blkh) );
81 const unsigned int blockSizeBytes = (
unsigned int)(
82 bandsProperties[ bandIdx ]->m_blkw *
83 bandsProperties[ bandIdx ]->m_blkh *
86 if( maxBlockSizeBytes < blockSizeBytes )
87 maxBlockSizeBytes = blockSizeBytes;
89 numbersOfBlocksX.push_back( (
unsigned int)bandsProperties[ bandIdx ]->m_nblocksx );
90 numbersOfBlocksY.push_back( (
unsigned int)bandsProperties[ bandIdx ]->m_nblocksy );
91 blocksSizesBytes.push_back( blockSizeBytes );
99 const double freeVMem = ( ((double)maxMemPercentUsed) / 100.0 ) *
100 std::min( totalPhysMem, ( ( totalVMem <= usedVMem ) ? 0.0 : ( totalVMem - usedVMem ) ) );
101 const unsigned int maxNumberOfBlocks = (
unsigned int)
102 std::max( 1.0, std::ceil( freeVMem / ((
double)maxBlockSizeBytes) ) );
105 numbersOfBlocksY, blocksSizesBytes,
107 throw Exception(
TR_MEMORY(
"Cannot initialize the blocks menager") );
109 for(
unsigned int bandsIdx = 0 ; bandsIdx < bandsProperties.size() ;
113 *(bandsProperties[ bandsIdx ]) , bandsIdx ) );
114 delete ( bandsProperties[ bandsIdx ] );
121 const std::vector<te::rst::BandProperty*> bandsProperties,
122 const unsigned int maxNumberOfRAMBlocks )
127 std::vector< unsigned int> numbersOfBlocksX;
128 std::vector< unsigned int> numbersOfBlocksY;
129 std::vector< unsigned int> blocksSizesBytes;
132 for(
unsigned int bandIdx = 0 ; bandIdx < bandsProperties.size() ; ++bandIdx )
135 bandsProperties[ bandIdx ]->m_nblocksx = bandsProperties[ 0 ]->m_nblocksx;
136 bandsProperties[ bandIdx ]->m_nblocksy = bandsProperties[ 0 ]->m_nblocksy;
137 bandsProperties[ bandIdx ]->m_blkw = bandsProperties[ 0 ]->m_blkw;
138 bandsProperties[ bandIdx ]->m_blkh = bandsProperties[ 0 ]->m_blkh;
139 bandsProperties[ bandIdx ]->m_nblocksx = (int)std::ceil(
141 ((
double)bandsProperties[ bandIdx ]->m_blkw) );
142 bandsProperties[ bandIdx ]->m_nblocksy = (int)std::ceil(
144 ((
double)bandsProperties[ bandIdx ]->m_blkh) );
146 const unsigned int blockSizeBytes = (
unsigned int)(
147 bandsProperties[ bandIdx ]->m_blkw *
148 bandsProperties[ bandIdx ]->m_blkh *
151 numbersOfBlocksX.push_back( (
unsigned int)bandsProperties[ bandIdx ]->m_nblocksx );
152 numbersOfBlocksY.push_back( (
unsigned int)bandsProperties[ bandIdx ]->m_nblocksy );
153 blocksSizesBytes.push_back( blockSizeBytes );
158 numbersOfBlocksY, blocksSizesBytes,
160 throw Exception(
TR_MEMORY(
"Cannot initialize the blocks menager") );
162 for(
unsigned int bandsIdx = 0 ; bandsIdx < bandsProperties.size() ;
166 *(bandsProperties[ bandsIdx ]) , bandsIdx ) );
167 delete ( bandsProperties[ bandsIdx ] );
185 return std::map<std::string, std::string>();
190 std::vector<te::rst::BandProperty*> bandsProperties;
191 for(
unsigned int bandsIdx = 0 ; bandsIdx < m_bands.size() ; ++bandsIdx )
194 *(m_bands[ bandsIdx ]->getProperty() ) ) );
198 m_blocksManager.getMaxNumberOfRAMBlocks() );
203 if( m_bands.empty() )
return false;
207 const unsigned int blockExpansionSize = (
unsigned int)std::ceil(
208 ((
double)number) / ((
double)m_bands[ 0 ]->getProperty()->m_blkh) );
210 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > addedBlocksCoords;
212 for(
unsigned int bandsIdx = 0 ; bandsIdx < m_bands.size() ; ++bandsIdx )
214 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > bandAddedBlocksCoords;
215 if( ! m_blocksManager.addTopBlocks( blockExpansionSize, bandsIdx,
216 bandAddedBlocksCoords ) )
219 addedBlocksCoords.insert( addedBlocksCoords.end(), bandAddedBlocksCoords.begin(),
220 bandAddedBlocksCoords.end() );
222 m_bands[ bandsIdx ]->getProperty()->m_nblocksy += blockExpansionSize;
225 dummyFillBlocks( addedBlocksCoords );
227 const unsigned int newLinesNumber =
228 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkh) *
229 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_nblocksy );
230 const unsigned int newColsNumber =
231 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkw) *
232 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_nblocksx );
234 const double addedLinesNumber = (double)( blockExpansionSize *
235 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkh) );
238 m_grid->getExtent()->getLowerLeftX(),
239 m_grid->getExtent()->getUpperRightY() + ( addedLinesNumber *
240 m_grid->getResolutionY() ) );
243 newLinesNumber, m_grid->getResolutionX(), m_grid->getResolutionY(),
244 &newULC, m_grid->getSRID() ) );
254 if( m_bands.empty() )
return false;
258 const unsigned int currExtraLines =
259 ( (
unsigned int)( m_bands[ 0 ]->getProperty()->m_blkh *
260 m_bands[ 0 ]->getProperty()->m_nblocksy ) ) -
261 m_grid->getNumberOfRows();
264 m_grid->getExtent()->getLowerLeftX(),
265 m_grid->getExtent()->getUpperRightY() );
267 if( currExtraLines < number )
269 const unsigned int blockExpansionSize = (
unsigned int)std::ceil(
270 ((
double)number) / ((
double)m_bands[ 0 ]->getProperty()->m_blkh) );
272 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > addedBlocksCoords;
274 for(
unsigned int bandsIdx = 0 ; bandsIdx < m_bands.size() ; ++bandsIdx )
276 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > bandAddedBlocksCoords;
277 if( ! m_blocksManager.addBottomBlocks( blockExpansionSize, bandsIdx,
278 bandAddedBlocksCoords ) )
281 addedBlocksCoords.insert( addedBlocksCoords.end(), bandAddedBlocksCoords.begin(),
282 bandAddedBlocksCoords.end() );
284 m_bands[ bandsIdx ]->getProperty()->m_nblocksy += blockExpansionSize;
287 dummyFillBlocks( addedBlocksCoords );
289 const unsigned int newLinesNumber =
290 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkh) *
291 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_nblocksy );
292 const unsigned int newColsNumber =
293 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkw) *
294 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_nblocksx );
297 newLinesNumber, m_grid->getResolutionX(), m_grid->getResolutionY(),
298 &uLC, m_grid->getSRID() ) );
305 m_grid->getNumberOfColumns(),
306 m_grid->getNumberOfRows() + number,
307 m_grid->getResolutionX(), m_grid->getResolutionY(),
308 &uLC, m_grid->getSRID() ) );
319 if( m_bands.empty() )
return false;
323 const unsigned int oldColsNumber =
324 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkw) *
325 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_nblocksx );
327 const unsigned int blockExpansionSize = (
unsigned int)std::ceil(
328 ((
double)number) / ((
double)m_bands[ 0 ]->getProperty()->m_blkw) );
330 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > addedBlocksCoords;
332 for(
unsigned int bandsIdx = 0 ; bandsIdx < m_bands.size() ; ++bandsIdx )
334 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > bandAddedBlocksCoords;
335 if( ! m_blocksManager.addLeftBlocks( blockExpansionSize, bandsIdx,
336 bandAddedBlocksCoords ) )
339 addedBlocksCoords.insert( addedBlocksCoords.end(), bandAddedBlocksCoords.begin(),
340 bandAddedBlocksCoords.end() );
342 m_bands[ bandsIdx ]->getProperty()->m_nblocksx += blockExpansionSize;
345 dummyFillBlocks( addedBlocksCoords );
347 const unsigned int newLinesNumber =
348 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkh) *
349 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_nblocksy );
350 const unsigned int newColsNumber =
351 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkw) *
352 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_nblocksx );
354 const double addedColsNumber = (double)( newColsNumber - oldColsNumber );
357 m_grid->getExtent()->getLowerLeftX() - ( addedColsNumber *
358 m_grid->getResolutionX() ),
359 m_grid->getExtent()->getUpperRightY() );
362 newLinesNumber, m_grid->getResolutionX(), m_grid->getResolutionY(),
363 &newULC, m_grid->getSRID() ) );
373 if( m_bands.empty() )
return false;
377 const unsigned int currExtraCols =
378 ( (
unsigned int)( m_bands[ 0 ]->getProperty()->m_blkw *
379 m_bands[ 0 ]->getProperty()->m_nblocksx ) ) -
380 m_grid->getNumberOfColumns();
383 m_grid->getExtent()->getLowerLeftX(),
384 m_grid->getExtent()->getUpperRightY() );
386 if( currExtraCols < number )
388 const unsigned int blockExpansionSize = (
unsigned int)std::ceil(
389 ((
double)number) / ((
double)m_bands[ 0 ]->getProperty()->m_blkw) );
391 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > addedBlocksCoords;
393 for(
unsigned int bandsIdx = 0 ; bandsIdx < m_bands.size() ; ++bandsIdx )
395 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > bandAddedBlocksCoords;
396 if( ! m_blocksManager.addRightBlocks( blockExpansionSize, bandsIdx,
397 bandAddedBlocksCoords ) )
400 addedBlocksCoords.insert( addedBlocksCoords.end(), bandAddedBlocksCoords.begin(),
401 bandAddedBlocksCoords.end() );
403 m_bands[ bandsIdx ]->getProperty()->m_nblocksx += blockExpansionSize;
406 dummyFillBlocks( addedBlocksCoords );
408 const unsigned int newLinesNumber =
409 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkh) *
410 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_nblocksy );
411 const unsigned int newColsNumber =
412 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_blkw) *
413 ((
unsigned int)m_bands[ 0 ]->getProperty()->m_nblocksx );
416 newLinesNumber, m_grid->getResolutionX(), m_grid->getResolutionY(),
417 &uLC, m_grid->getSRID() ) );
424 m_grid->getNumberOfColumns() + number,
425 m_grid->getNumberOfRows(),
426 m_grid->getResolutionX(), m_grid->getResolutionY(),
427 &uLC, m_grid->getSRID() ) );
438 if( m_bands.empty() )
return false;
442 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > addedBlocksCoords;
444 if( ! m_blocksManager.addTopBands( number, addedBlocksCoords ) )
447 m_bands.insert( m_bands.begin(), number, 0 );
448 for(
unsigned int bIdx = 0 ; bIdx < m_bands.size() ; ++bIdx )
450 if( m_bands[ bIdx ] )
455 delete( oldBandPtr );
460 *( m_bands[ number ]->getProperty() ), bIdx );
464 dummyFillBlocks( addedBlocksCoords );
472 if( m_bands.empty() )
return false;
476 std::vector< ExpansibleBandBlocksManager::BlockIndex3D > addedBlocksCoords;
478 if( ! m_blocksManager.addBottomBands( number, addedBlocksCoords ) )
481 unsigned int lastBandIdx = (
unsigned int)m_bands.size() - 1;
482 m_bands.insert( m_bands.end(), number, 0 );
483 for(
unsigned int bIdx = lastBandIdx + 1 ; bIdx < m_bands.size() ; ++bIdx )
486 *( m_bands[ lastBandIdx ]->getProperty() ), bIdx );
489 dummyFillBlocks( addedBlocksCoords );
497 if( m_bands.size() > 0 )
499 for(
unsigned int bandsIdx = 0 ; bandsIdx < m_bands.size() ; ++bandsIdx )
500 delete (m_bands[ bandsIdx ]);
504 m_blocksManager.free();
509 for(
unsigned int bandsIdx = 0 ; bandsIdx < m_bands.size() ;
525 unsigned int blockXIdx = 0;
526 unsigned int blockYIdx = 0;
530 for( blockXIdx = 0 ; blockXIdx < nBlocksX ; ++blockXIdx )
532 for( blockYIdx = 0 ; blockYIdx < nBlocksY ; ++blockYIdx )
534 blockPtr = m_blocksManager.getBlockPointer( bandsIdx, blockXIdx,
538 for( eIdx = 0 ; eIdx < elementsNumber ; ++eIdx )
539 sb( eIdx, blockPtr, noDataValuePtr );
546 const std::vector< ExpansibleBandBlocksManager::BlockIndex3D >& blocksCoords )
548 for(
unsigned int coodsIdx = 0 ; coodsIdx < blocksCoords.size() ;
563 void* blockPtr = m_blocksManager.getBlockPointer( block3DIdx.
m_dim0Index,
566 const int elementsNumber = band.getProperty()->m_blkh *
567 band.getProperty()->m_blkw;
569 double* noDataValuePtr = &( band.getProperty()->m_noDataValue );
571 for(
int eIdx = 0 ; eIdx < elementsNumber ; ++eIdx )
572 sb( eIdx, blockPtr, noDataValuePtr );
int m_blkh
Block height (pixels).
void dummyFillAllBlocks()
Fill all blocks with dummy values.
te::dt::AbstractData * clone() const
It returns a clone of this object.
void dummyFillBlocks(const std::vector< ExpansibleBandBlocksManager::BlockIndex3D > &blocksCoords)
Fill the required blocks with dummy values.
TERASTEREXPORT int GetPixelSize(int datatype)
Returns the byte size of a given datatype.
A raster (stored in memory and eventually swapped to disk) where it is possible to dynamically add li...
std::vector< ExpansibleBand * > m_bands
Internal raster bands.
bool initialize(const unsigned int maxNumberRAMBlocks, const std::vector< unsigned int > &numbersOfBlocksX, const std::vector< unsigned int > &numbersOfBlocksY, const std::vector< unsigned int > &blocksSizesBytes, const unsigned long int maxDiskFilesSize)
Initialize this instance to an initial state.
void(* GetBufferValueFPtr)(int index, void *buffer, double *value)
The type of function used to extract data from a buffer.
A raster (stored in memory and eventually swapped to disk) where it is possible to dynamically add li...
#define TR_MEMORY(message)
#define TLINTERNAL_EXPANSIBLERASTER_MAXDISKFILESSIZE
An exception class for the TerraLib In-Memory Data Access driver.
bool addTopBands(const unsigned int number)
New bands will be added at the top of the raster (before the first band).
int m_nblocksy
The number of blocks in y.
void(* SetBufferValueFPtr)(int index, void *buffer, const double *value)
The type of function used to extract data from a buffer.
An utility struct for representing 2D coordinates.
CoordDataType m_dim0Index
Block Z (band) index.
unsigned int getNumberOfRows() const
Returns the grid number of rows.
AccessPolicy
Supported data access policies (can be used as bitfield).
A raster class for memory.
void open(const std::map< std::string, std::string > &rinfo, te::common::AccessPolicy p=te::common::RAccess)
Opens a raster.
int m_type
The data type of the elements in the band.
A rectified grid is the spatial support for raster data.
int m_blkw
Block width (pixels).
TERASTEREXPORT void SetBlockFunctions(GetBufferValueFPtr *gb, GetBufferValueFPtr *gbi, SetBufferValueFPtr *sb, SetBufferValueFPtr *sbi, int type)
Sets the pointers to functions that helps to extract a double or complex value from a specific buffer...
std::map< std::string, std::string > getInfo() const
It returns additional information about the raster.
bool addRightColumns(const unsigned int number)
New columns will be added at the right of the raster (after the last column).
void free()
Free all allocated internal resources and go back to the initial state.
unsigned long int GetTotalPhysicalMemory()
Returns the amount of total physical memory (bytes).
bool addBottomLines(const unsigned int number)
New lines will be added at the bottom of the raster (after de the last line).
A base class for values that can be retrieved from the data access module.
A raster band description.
double m_noDataValue
Value to indicate elements where there is no data, default is std::numeric_limits<double>::max().
CoordDataType m_dim2Index
Block X index.
bool addBottomBands(const unsigned int number)
New bands will be added at the bottom of the raster (after de the last band).
int m_nblocksx
The number of blocks in x.
BandProperty * getProperty()
Returns the band property.
unsigned int getNumberOfColumns() const
Returns the grid number of columns.
unsigned long int GetUsedVirtualMemory()
Returns the amount of used virtual memory (bytes) for the current process (physical + swapped)...
unsigned long int GetTotalVirtualMemory()
Returns the amount of total virtual memory (bytes) that can be claimed by the current process (physic...
bool addLeftColumns(const unsigned int number)
New columns will be added at the left of the raster (before the first column).
CoordDataType m_dim1Index
Block Y index.
ExpansibleBandBlocksManager m_blocksManager
Internal blocks manager.
bool addTopLines(const unsigned int number)
New lines will be added at the top of the raster (before the first line).