27 #include "../common/PlatformUtils.h"
28 #include "../raster/Band.h"
29 #include "../raster/BandProperty.h"
58 const te::rst::Raster& externalRaster,
const unsigned char maxMemPercentUsed,
59 const unsigned int dataPrefetchThreshold)
65 unsigned int maxBlockSizeBytes = 0;
67 for(
unsigned int bandIdx = 0 ; bandIdx < externalRaster.
getNumberOfBands() ;
78 const double freeVMem = ( ((double)maxMemPercentUsed) / 100.0 ) *
79 std::min( totalPhysMem, ( ( totalVMem <= usedVMem ) ? 0.0 : ( totalVMem - usedVMem ) ) );
80 const unsigned int maxNumberOfCacheBlocks = (
unsigned int)
81 std::max( 1.0, std::ceil( freeVMem / ((
double)maxBlockSizeBytes) ) );
83 return initialize( maxNumberOfCacheBlocks, externalRaster, dataPrefetchThreshold );
87 const unsigned int maxNumberOfCacheBlocks,
89 const unsigned int dataPrefetchThreshold)
95 unsigned int numberOfRasterBlocks = 0;
97 for(
unsigned int bandIdx = 0 ; bandIdx < externalRaster.
getNumberOfBands() ;
106 if( m_globalBlockSizeBytes < (
unsigned int)externalRaster.
getBand( bandIdx )->
getBlockSize() )
109 numberOfRasterBlocks +=
117 m_dataPrefetchThreshold = dataPrefetchThreshold;
119 m_maxNumberOfCacheBlocks = std::min( maxNumberOfCacheBlocks, numberOfRasterBlocks );
120 m_maxNumberOfCacheBlocks = std::max( m_maxNumberOfCacheBlocks, (
unsigned int)1 );
122 unsigned int blockBIdx = 0;
125 for( blockBIdx = 0 ; blockBIdx < m_blocksPointers.size() ; ++blockBIdx )
127 m_blocksPointers[ blockBIdx ].resize( m_globalBlocksNumberY );
129 for(
unsigned int blockYIdx = 0 ; blockYIdx < m_globalBlocksNumberY ;
132 m_blocksPointers[ blockBIdx ][ blockYIdx ].resize( m_globalBlocksNumberX, 0 );
136 if( m_dataPrefetchThreshold )
139 m_threadParams.m_dataPrefetchThreshold = dataPrefetchThreshold;
140 m_threadParams.m_taskFinished =
false;
141 m_threadParams.m_task = ThreadParameters::InvalidTaskT;
142 m_threadParams.m_blockPtr= 0;
143 m_threadParams.m_exchangeBlockPtr= 0;
144 m_threadParams.m_blockB = 0;
145 m_threadParams.m_blockX = 0;
146 m_threadParams.m_blockY = 0;
147 m_threadParams.m_threadDataBlockHandler.reset(
new unsigned char[
148 m_globalBlockSizeBytes ] );
150 m_threadHandler.reset(
new boost::thread( threadEntry, &m_threadParams ) );
160 if( m_dataPrefetchThreshold )
163 boost::lock_guard<boost::mutex> lock(m_threadParams.m_doTaskMutex);
164 m_threadParams.m_task = ThreadParameters::SuicideTastT;
165 m_threadParams.m_doTaskCondVar.notify_one();
168 m_threadHandler->join();
169 m_threadHandler.reset();
174 if( ( m_rasterPtr != 0 ) &&
177 unsigned int blockBIdx = 0;
178 unsigned int blockYIdx = 0;
179 unsigned int blockXIdx = 0;
182 for( blockBIdx = 0 ; blockBIdx < m_blocksPointers.size() ; ++blockBIdx )
184 for( blockYIdx = 0 ; blockYIdx < m_globalBlocksNumberY ;
187 for( blockXIdx = 0 ; blockXIdx < m_globalBlocksNumberX ;
190 blockPtr = m_blocksPointers[ blockBIdx ][ blockYIdx ][ blockXIdx ];
194 m_rasterPtr->getBand( blockBIdx )->write( blockXIdx, blockYIdx,
202 m_blocksPointers.clear();
204 unsigned char* blockPtr = 0;
205 for( std::vector< unsigned char* >::size_type blocksHandlerIdx = 0 ;
206 blocksHandlerIdx < m_blocksHandler.size() ; ++blocksHandlerIdx )
208 blockPtr = m_blocksHandler[ blocksHandlerIdx ];
211 delete[]( blockPtr );
214 m_blocksHandler.clear();
216 m_blocksFifo.clear();
217 m_threadHandler.reset();
218 m_threadParams.m_threadDataBlockHandler.reset();
224 unsigned int x,
unsigned int y )
226 assert( m_rasterPtr );
227 assert( band < m_rasterPtr->getNumberOfBands() );
228 assert( x < m_globalBlocksNumberX );
229 assert( y < m_globalBlocksNumberY );
230 assert( x < (
unsigned int)m_rasterPtr->getBand( band )->getProperty()->m_nblocksx );
231 assert( y < (
unsigned int)m_rasterPtr->getBand( band )->getProperty()->m_nblocksy );
233 m_getBlockPointer_BlkPtr = m_blocksPointers[ band ][ y ][ x ];
235 if( m_getBlockPointer_BlkPtr == 0 )
238 if( m_blocksHandler.size() < m_maxNumberOfCacheBlocks )
240 m_getBlockPointer_BlkPtr =
new unsigned char[ m_globalBlockSizeBytes ];
241 m_blocksHandler.push_back( m_getBlockPointer_BlkPtr );
245 newBlockFifoIndex.
m_b = band;
246 newBlockFifoIndex.
m_y = y;
247 newBlockFifoIndex.
m_x = x;
248 m_blocksFifo.push_back( newBlockFifoIndex );
252 BlockIndex& choosedSwapBlockIndex = m_blocksFifo[
253 m_blocksFifoNextSwapBlockIndex ];
255 m_getBlockPointer_BlkPtr = m_blocksPointers[ choosedSwapBlockIndex.
m_b ][
256 choosedSwapBlockIndex.
m_y ][ choosedSwapBlockIndex.
m_x ];
257 assert( m_getBlockPointer_BlkPtr );
258 m_blocksPointers[ choosedSwapBlockIndex.
m_b ][
259 choosedSwapBlockIndex.
m_y ][ choosedSwapBlockIndex.
m_x ] = 0;
264 if( m_dataPrefetchThreshold )
268 boost::lock_guard<boost::mutex> lock(m_threadParams.m_doTaskMutex);
270 m_threadParams.m_blockB = choosedSwapBlockIndex.
m_b;
271 m_threadParams.m_blockX = choosedSwapBlockIndex.
m_x;
272 m_threadParams.m_blockY = choosedSwapBlockIndex.
m_y;
273 m_threadParams.m_blockPtr = m_getBlockPointer_BlkPtr;
274 m_threadParams.m_exchangeBlockPtr = 0;
275 m_threadParams.m_task = ThreadParameters::WriteTaskT;
276 m_threadParams.m_taskFinished =
false;
279 m_threadParams.m_doTaskCondVar.notify_one();
282 while( ! m_threadParams.m_taskFinished )
284 boost::unique_lock<boost::mutex> lock(
285 m_threadParams.m_taskFinishedMutex);
287 while( ! m_threadParams.m_taskFinished )
289 m_threadParams.m_taskFinishedCondVar.wait( lock );
295 m_rasterPtr->getBand( choosedSwapBlockIndex.
m_b )->write(
296 choosedSwapBlockIndex.
m_x, choosedSwapBlockIndex.
m_y,
297 m_getBlockPointer_BlkPtr );
302 choosedSwapBlockIndex.
m_b = band;
303 choosedSwapBlockIndex.
m_y = y;
304 choosedSwapBlockIndex.
m_x = x;
305 m_blocksFifoNextSwapBlockIndex = ( m_blocksFifoNextSwapBlockIndex + 1 ) %
306 ((
unsigned int)m_blocksFifo.size());
310 if( m_dataPrefetchThreshold )
314 boost::lock_guard<boost::mutex> lock(m_threadParams.m_doTaskMutex);
316 m_threadParams.m_blockB = band;
317 m_threadParams.m_blockX = x;
318 m_threadParams.m_blockY = y;
319 m_threadParams.m_blockPtr = 0;
320 m_threadParams.m_exchangeBlockPtr = m_getBlockPointer_BlkPtr;
321 m_threadParams.m_task = ThreadParameters::ReadTaskT;
322 m_threadParams.m_taskFinished =
false;
324 m_threadParams.m_doTaskCondVar.notify_one();
329 boost::unique_lock<boost::mutex> lock(
330 m_threadParams.m_taskFinishedMutex);
332 while( ! m_threadParams.m_taskFinished )
334 m_threadParams.m_taskFinishedCondVar.wait( lock );
338 m_threadParams.m_taskFinished =
false;
340 m_blocksPointers[ band ][ y ][ x ] = m_threadParams.m_blockPtr;
341 m_getBlockPointer_BlkPtr = m_threadParams.m_blockPtr;
345 m_rasterPtr->getBand( band )->read( x, y, m_getBlockPointer_BlkPtr );
346 m_blocksPointers[ band ][ y ][ x ] = m_getBlockPointer_BlkPtr;
352 return m_getBlockPointer_BlkPtr;
364 assert( internalExchangeBlockPtr );
366 int globalReadAheadDirectionVectorX = 0;
367 int globalReadAheadDirectionVectorY = 0;
368 int globalReadAheadDirectionVectorB = 0;
369 bool globalReadAheadBlockReady =
false;
370 int globalReadAheadBlockB = 0;
371 int globalReadAheadBlockX = 0;
372 int globalReadAheadBlockY = 0;
373 int lastReadBlockX = 0;
374 int lastReadBlockY = 0;
375 int lastReadBlockB = 0;
377 int currentReadDirectionVectorX = 0;
378 int currentReadDirectionVectorY = 0;
379 int currentReadDirectionVectorB = 0;
380 int nextReadAheadBlkB = 0;
381 int nextReadAheadBlkX = 0;
382 int nextReadAheadBlkY = 0;
383 bool doReadAhead =
false;
388 boost::unique_lock<boost::mutex> lock( paramsPtr->
m_doTaskMutex );
389 while( paramsPtr->
m_task == ThreadParameters::InvalidTaskT )
394 if( paramsPtr->
m_task == ThreadParameters::ReadTaskT )
401 if( globalReadAheadBlockReady
402 && ( globalReadAheadBlockB == (
int)paramsPtr->
m_blockB )
403 && ( globalReadAheadBlockX == (
int)paramsPtr->
m_blockX )
404 && ( globalReadAheadBlockY == (
int)paramsPtr->
m_blockY ) )
409 paramsPtr->
m_blockPtr = internalExchangeBlockPtr;
410 globalReadAheadBlockReady =
false;
433 paramsPtr->
m_task = ThreadParameters::InvalidTaskT;
442 currentReadDirectionVectorX = ( ((int)paramsPtr->
m_blockX) - lastReadBlockX );
443 currentReadDirectionVectorY = ( ((int)paramsPtr->
m_blockY) - lastReadBlockY );
444 currentReadDirectionVectorB = ( ((int)paramsPtr->
m_blockB) - lastReadBlockB );
448 if( ( std::abs( currentReadDirectionVectorX ) < 2 ) &&
449 ( std::abs( currentReadDirectionVectorY ) < 2 )
450 && ( std::abs( currentReadDirectionVectorB ) < 2 ) )
452 globalReadAheadDirectionVectorX += currentReadDirectionVectorX;
453 globalReadAheadDirectionVectorY += currentReadDirectionVectorY;
454 globalReadAheadDirectionVectorB += currentReadDirectionVectorB;
456 if( ((
unsigned int)std::abs( globalReadAheadDirectionVectorB )) >
459 if( globalReadAheadDirectionVectorB > 0 )
461 nextReadAheadBlkB = ((int)paramsPtr->
m_blockB) + 1;
462 --globalReadAheadDirectionVectorB;
466 nextReadAheadBlkB = ((int)paramsPtr->
m_blockB) - 1;
467 ++globalReadAheadDirectionVectorB;
470 if( ( nextReadAheadBlkB >= 0 ) &&
474 nextReadAheadBlkB = ((int)paramsPtr->
m_blockB);
477 if( ((
unsigned int)std::abs( globalReadAheadDirectionVectorX )) >
480 if( globalReadAheadDirectionVectorX > 0 )
482 nextReadAheadBlkX = ((int)paramsPtr->
m_blockX) + 1;
483 --globalReadAheadDirectionVectorX;
487 nextReadAheadBlkX = ((int)paramsPtr->
m_blockX) - 1;
488 ++globalReadAheadDirectionVectorX;
491 if( ( nextReadAheadBlkX >= 0 ) &&
492 ( nextReadAheadBlkX <
496 nextReadAheadBlkX = ((int)paramsPtr->
m_blockX);
499 if( ((
unsigned int)std::abs( globalReadAheadDirectionVectorY )) >
502 if( globalReadAheadDirectionVectorY > 0 )
504 nextReadAheadBlkY = ((int)paramsPtr->
m_blockY) + 1;
505 --globalReadAheadDirectionVectorY;
509 nextReadAheadBlkY = ((int)paramsPtr->
m_blockY) - 1;
510 ++globalReadAheadDirectionVectorY;
513 if( ( nextReadAheadBlkY >= 0 ) &&
514 ( nextReadAheadBlkY <
518 nextReadAheadBlkY = ((int)paramsPtr->
m_blockY);
523 globalReadAheadDirectionVectorX = 0;
524 globalReadAheadDirectionVectorY = 0;
525 globalReadAheadDirectionVectorB = 0;
533 assert( nextReadAheadBlkX <
535 assert( nextReadAheadBlkY <
539 nextReadAheadBlkY, internalExchangeBlockPtr );
541 globalReadAheadBlockX = nextReadAheadBlkX;
542 globalReadAheadBlockY = nextReadAheadBlkY;
543 globalReadAheadBlockB = nextReadAheadBlkB;
544 globalReadAheadBlockReady =
true;
550 lastReadBlockX = (int)paramsPtr->
m_blockX;
551 lastReadBlockY = (
int)paramsPtr->
m_blockY;
552 lastReadBlockB = (int)paramsPtr->
m_blockB;
554 else if( paramsPtr->
m_task == ThreadParameters::WriteTaskT )
567 paramsPtr->
m_task = ThreadParameters::InvalidTaskT;
570 else if( paramsPtr->
m_task == ThreadParameters::SuicideTastT )
unsigned int m_x
Block index over the X axis.
unsigned int m_blockB
Raster block band index.
unsigned int m_globalBlocksNumberY
The maximum number of blocks (Y axis) for all bands.
unsigned int m_maxNumberOfCacheBlocks
The maximum number of cache blocks.
int getBlockSize() const
It returns the number of bytes ocuppied by a data block.
unsigned int m_blocksFifoNextSwapBlockIndex
The next block swapp index over m_blocksFifo.
unsigned int m_dataPrefetchThreshold
The user defined read-ahead threshold.
unsigned int m_globalBlocksNumberX
The maximum number of blocks (X axis) for all bands.
RAM cached and tiled raster band blocks manager.
unsigned char * m_exchangeBlockPtr
Exchange block pointer.
boost::mutex m_taskFinishedMutex
used by the thread to inform a task finishment.
void read(int x, int y, void *buffer) const
It reads a data block to the specified buffer.
bool m_taskFinished
true when the thread has finished the required task.
unsigned int m_blockY
Raster block Y index.
void initState()
Initialize this instance to an initial state.
int m_nblocksy
The number of blocks in y.
virtual std::size_t getNumberOfBands() const =0
Returns the number of bands (dimension of cells attribute values) in the raster.
unsigned char * m_blockPtr
Input block pointer.
unsigned int m_globalBlockSizeBytes
The maximum block size for all bands.
unsigned int m_y
Block index over the Y axis.
unsigned char * m_getBlockPointer_BlkPtr
TaskType m_task
The required task to be performed (read/write/exit).
bool initialize(const te::rst::Raster &externalRaster, const unsigned char maxMemPercentUsed, const unsigned int dataPrefetchThreshold)
Initialize this instance to an initial state.
boost::mutex m_doTaskMutex
Used when wakenning the thread to perform some task.
te::rst::Raster * m_rasterPtr
External raster pointer.
unsigned long int GetTotalPhysicalMemory()
Returns the amount of total physical memory (bytes).
unsigned int m_blockX
Raster block X index.
~CachedBandBlocksManager()
unsigned int m_dataPrefetchThreshold
The read-ahead data prefetch threshold (0-will disable prefetch, 1-data always prefetched, higher values will do prefetch when necessary).
boost::condition_variable m_taskFinishedCondVar
Used to wait for the required task finishment.
te::rst::Raster * m_rasterPtr
External raster pointer.
Internal read/write thread execution parameters.
int m_nblocksx
The number of blocks in x.
An abstract class for raster data strucutures.
BandProperty * getProperty()
Returns the band property.
virtual const Band * getBand(std::size_t i) const =0
Returns the raster i-th band.
unsigned long int GetUsedVirtualMemory()
Returns the amount of used virtual memory (bytes) for the current process (physical + swapped)...
unsigned long int GetTotalVirtualMemory()
Returns the amount of total virtual memory (bytes) that can be claimed by the current process (physic...
static void threadEntry(ThreadParameters *paramsPtr)
Thread entry.
void write(int x, int y, void *buffer)
It writes a data block from the specified buffer.
boost::condition_variable m_doTaskCondVar
Used by the thread when awakenning to perform some task.
boost::scoped_array< unsigned char > m_threadDataBlockHandler
A extra block used in exchange when a read-ahead task is performed.
unsigned int m_b
Block band index.
CachedBandBlocksManager()
void * getBlockPointer(unsigned int band, unsigned int x, unsigned int y)
Returns a pointer to the required data block.