27 #include "../common/PlatformUtils.h"
28 #include "../raster/Band.h"
29 #include "../raster/BandProperty.h"
58 const te::rst::Raster& externalRaster,
const unsigned char maxMemPercentUsed,
59 const unsigned int dataPrefetchThreshold)
65 unsigned int maxBlockSizeBytes = 0;
67 for(
unsigned int bandIdx = 0 ; bandIdx < externalRaster.
getNumberOfBands() ;
77 const double freeVMem = ( ((double)maxMemPercentUsed) / 100.0 ) *
78 std::min( totalPhysMem, ( totalVMem - usedVMem ) );
79 const unsigned int maxNumberOfCacheBlocks = (
unsigned int)
80 std::max( 1.0, std::ceil( freeVMem / ((
double)maxBlockSizeBytes) ) );
82 return initialize( maxNumberOfCacheBlocks, externalRaster, dataPrefetchThreshold );
86 const unsigned int maxNumberOfCacheBlocks,
88 const unsigned int dataPrefetchThreshold)
94 unsigned int numberOfRasterBlocks = 0;
96 for(
unsigned int bandIdx = 0 ; bandIdx < externalRaster.
getNumberOfBands() ;
105 if( m_globalBlockSizeBytes < (
unsigned int)externalRaster.
getBand( bandIdx )->
getBlockSize() )
108 numberOfRasterBlocks +=
116 m_dataPrefetchThreshold = dataPrefetchThreshold;
118 m_maxNumberOfCacheBlocks = std::min( maxNumberOfCacheBlocks, numberOfRasterBlocks );
119 m_maxNumberOfCacheBlocks = std::max( m_maxNumberOfCacheBlocks, (
unsigned int)1 );
121 unsigned int blockBIdx = 0;
124 for( blockBIdx = 0 ; blockBIdx < m_blocksPointers.size() ; ++blockBIdx )
126 m_blocksPointers[ blockBIdx ].resize( m_globalBlocksNumberY );
128 for(
unsigned int blockYIdx = 0 ; blockYIdx < m_globalBlocksNumberY ;
131 m_blocksPointers[ blockBIdx ][ blockYIdx ].resize( m_globalBlocksNumberX, 0 );
135 if( m_dataPrefetchThreshold )
138 m_threadParams.m_dataPrefetchThreshold = dataPrefetchThreshold;
139 m_threadParams.m_taskFinished =
false;
140 m_threadParams.m_task = ThreadParameters::InvalidTaskT;
141 m_threadParams.m_blockPtr= 0;
142 m_threadParams.m_exchangeBlockPtr= 0;
143 m_threadParams.m_blockB = 0;
144 m_threadParams.m_blockX = 0;
145 m_threadParams.m_blockY = 0;
146 m_threadParams.m_threadDataBlockHandler.reset(
new unsigned char[
147 m_globalBlockSizeBytes ] );
149 m_threadHandler.reset(
new boost::thread( threadEntry, &m_threadParams ) );
159 if( m_dataPrefetchThreshold )
162 boost::lock_guard<boost::mutex> lock(m_threadParams.m_doTaskMutex);
163 m_threadParams.m_task = ThreadParameters::SuicideTastT;
164 m_threadParams.m_doTaskCondVar.notify_one();
167 m_threadHandler->join();
168 m_threadHandler.reset();
173 if( ( m_rasterPtr != 0 ) &&
176 unsigned int blockBIdx = 0;
177 unsigned int blockYIdx = 0;
178 unsigned int blockXIdx = 0;
181 for( blockBIdx = 0 ; blockBIdx < m_blocksPointers.size() ; ++blockBIdx )
183 for( blockYIdx = 0 ; blockYIdx < m_globalBlocksNumberY ;
186 for( blockXIdx = 0 ; blockXIdx < m_globalBlocksNumberX ;
189 blockPtr = m_blocksPointers[ blockBIdx ][ blockYIdx ][ blockXIdx ];
193 m_rasterPtr->getBand( blockBIdx )->write( blockXIdx, blockYIdx,
201 m_blocksPointers.clear();
203 unsigned char* blockPtr = 0;
204 for( std::vector< unsigned char* >::size_type blocksHandlerIdx = 0 ;
205 blocksHandlerIdx < m_blocksHandler.size() ; ++blocksHandlerIdx )
207 blockPtr = m_blocksHandler[ blocksHandlerIdx ];
210 delete[]( blockPtr );
213 m_blocksHandler.clear();
215 m_blocksFifo.clear();
216 m_threadHandler.reset();
217 m_threadParams.m_threadDataBlockHandler.reset();
223 unsigned int x,
unsigned int y )
225 assert( m_rasterPtr );
226 assert( band < m_rasterPtr->getNumberOfBands() );
227 assert( x < m_globalBlocksNumberX );
228 assert( y < m_globalBlocksNumberY );
229 assert( x < (
unsigned int)m_rasterPtr->getBand( band )->getProperty()->m_nblocksx );
230 assert( y < (
unsigned int)m_rasterPtr->getBand( band )->getProperty()->m_nblocksy );
232 m_getBlockPointer_BlkPtr = m_blocksPointers[ band ][ y ][ x ];
234 if( m_getBlockPointer_BlkPtr == 0 )
237 if( m_blocksHandler.size() < m_maxNumberOfCacheBlocks )
239 m_getBlockPointer_BlkPtr =
new unsigned char[ m_globalBlockSizeBytes ];
240 m_blocksHandler.push_back( m_getBlockPointer_BlkPtr );
244 newBlockFifoIndex.
m_b = band;
245 newBlockFifoIndex.
m_y = y;
246 newBlockFifoIndex.
m_x = x;
247 m_blocksFifo.push_back( newBlockFifoIndex );
251 BlockIndex& choosedSwapBlockIndex = m_blocksFifo[
252 m_blocksFifoNextSwapBlockIndex ];
254 m_getBlockPointer_BlkPtr = m_blocksPointers[ choosedSwapBlockIndex.
m_b ][
255 choosedSwapBlockIndex.
m_y ][ choosedSwapBlockIndex.
m_x ];
256 assert( m_getBlockPointer_BlkPtr );
257 m_blocksPointers[ choosedSwapBlockIndex.
m_b ][
258 choosedSwapBlockIndex.
m_y ][ choosedSwapBlockIndex.
m_x ] = 0;
263 if( m_dataPrefetchThreshold )
267 boost::lock_guard<boost::mutex> lock(m_threadParams.m_doTaskMutex);
269 m_threadParams.m_blockB = choosedSwapBlockIndex.
m_b;
270 m_threadParams.m_blockX = choosedSwapBlockIndex.
m_x;
271 m_threadParams.m_blockY = choosedSwapBlockIndex.
m_y;
272 m_threadParams.m_blockPtr = m_getBlockPointer_BlkPtr;
273 m_threadParams.m_exchangeBlockPtr = 0;
274 m_threadParams.m_task = ThreadParameters::WriteTaskT;
275 m_threadParams.m_taskFinished =
false;
278 m_threadParams.m_doTaskCondVar.notify_one();
281 while( ! m_threadParams.m_taskFinished )
283 boost::unique_lock<boost::mutex> lock(
284 m_threadParams.m_taskFinishedMutex);
286 while( ! m_threadParams.m_taskFinished )
288 m_threadParams.m_taskFinishedCondVar.wait( lock );
294 m_rasterPtr->getBand( choosedSwapBlockIndex.
m_b )->write(
295 choosedSwapBlockIndex.
m_x, choosedSwapBlockIndex.
m_y,
296 m_getBlockPointer_BlkPtr );
301 choosedSwapBlockIndex.
m_b = band;
302 choosedSwapBlockIndex.
m_y = y;
303 choosedSwapBlockIndex.
m_x = x;
304 m_blocksFifoNextSwapBlockIndex = ( m_blocksFifoNextSwapBlockIndex + 1 ) %
305 ((
unsigned int)m_blocksFifo.size());
309 if( m_dataPrefetchThreshold )
313 boost::lock_guard<boost::mutex> lock(m_threadParams.m_doTaskMutex);
315 m_threadParams.m_blockB = band;
316 m_threadParams.m_blockX = x;
317 m_threadParams.m_blockY = y;
318 m_threadParams.m_blockPtr = 0;
319 m_threadParams.m_exchangeBlockPtr = m_getBlockPointer_BlkPtr;
320 m_threadParams.m_task = ThreadParameters::ReadTaskT;
321 m_threadParams.m_taskFinished =
false;
323 m_threadParams.m_doTaskCondVar.notify_one();
328 boost::unique_lock<boost::mutex> lock(
329 m_threadParams.m_taskFinishedMutex);
331 while( ! m_threadParams.m_taskFinished )
333 m_threadParams.m_taskFinishedCondVar.wait( lock );
337 m_threadParams.m_taskFinished =
false;
339 m_blocksPointers[ band ][ y ][ x ] = m_threadParams.m_blockPtr;
340 m_getBlockPointer_BlkPtr = m_threadParams.m_blockPtr;
344 m_rasterPtr->getBand( band )->read( x, y, m_getBlockPointer_BlkPtr );
345 m_blocksPointers[ band ][ y ][ x ] = m_getBlockPointer_BlkPtr;
351 return m_getBlockPointer_BlkPtr;
363 assert( internalExchangeBlockPtr );
365 int globalReadAheadDirectionVectorX = 0;
366 int globalReadAheadDirectionVectorY = 0;
367 int globalReadAheadDirectionVectorB = 0;
368 bool globalReadAheadBlockReady =
false;
369 int globalReadAheadBlockB = 0;
370 int globalReadAheadBlockX = 0;
371 int globalReadAheadBlockY = 0;
372 int lastReadBlockX = 0;
373 int lastReadBlockY = 0;
374 int lastReadBlockB = 0;
376 int currentReadDirectionVectorX = 0;
377 int currentReadDirectionVectorY = 0;
378 int currentReadDirectionVectorB = 0;
379 int nextReadAheadBlkB = 0;
380 int nextReadAheadBlkX = 0;
381 int nextReadAheadBlkY = 0;
382 bool doReadAhead =
false;
387 boost::unique_lock<boost::mutex> lock( paramsPtr->
m_doTaskMutex );
388 while( paramsPtr->
m_task == ThreadParameters::InvalidTaskT )
393 if( paramsPtr->
m_task == ThreadParameters::ReadTaskT )
400 if( globalReadAheadBlockReady
401 && ( globalReadAheadBlockB == (
int)paramsPtr->
m_blockB )
402 && ( globalReadAheadBlockX == (
int)paramsPtr->
m_blockX )
403 && ( globalReadAheadBlockY == (
int)paramsPtr->
m_blockY ) )
408 paramsPtr->
m_blockPtr = internalExchangeBlockPtr;
409 globalReadAheadBlockReady =
false;
432 paramsPtr->
m_task = ThreadParameters::InvalidTaskT;
441 currentReadDirectionVectorX = ( ((int)paramsPtr->
m_blockX) - lastReadBlockX );
442 currentReadDirectionVectorY = ( ((int)paramsPtr->
m_blockY) - lastReadBlockY );
443 currentReadDirectionVectorB = ( ((int)paramsPtr->
m_blockB) - lastReadBlockB );
447 if( ( std::abs( currentReadDirectionVectorX ) < 2 ) &&
448 ( std::abs( currentReadDirectionVectorY ) < 2 )
449 && ( std::abs( currentReadDirectionVectorB ) < 2 ) )
451 globalReadAheadDirectionVectorX += currentReadDirectionVectorX;
452 globalReadAheadDirectionVectorY += currentReadDirectionVectorY;
453 globalReadAheadDirectionVectorB += currentReadDirectionVectorB;
455 if( ((
unsigned int)std::abs( globalReadAheadDirectionVectorB )) >
458 if( globalReadAheadDirectionVectorB > 0 )
460 nextReadAheadBlkB = ((int)paramsPtr->
m_blockB) + 1;
461 --globalReadAheadDirectionVectorB;
465 nextReadAheadBlkB = ((int)paramsPtr->
m_blockB) - 1;
466 ++globalReadAheadDirectionVectorB;
469 if( ( nextReadAheadBlkB >= 0 ) &&
473 nextReadAheadBlkB = ((int)paramsPtr->
m_blockB);
476 if( ((
unsigned int)std::abs( globalReadAheadDirectionVectorX )) >
479 if( globalReadAheadDirectionVectorX > 0 )
481 nextReadAheadBlkX = ((int)paramsPtr->
m_blockX) + 1;
482 --globalReadAheadDirectionVectorX;
486 nextReadAheadBlkX = ((int)paramsPtr->
m_blockX) - 1;
487 ++globalReadAheadDirectionVectorX;
490 if( ( nextReadAheadBlkX >= 0 ) &&
491 ( nextReadAheadBlkX <
495 nextReadAheadBlkX = ((int)paramsPtr->
m_blockX);
498 if( ((
unsigned int)std::abs( globalReadAheadDirectionVectorY )) >
501 if( globalReadAheadDirectionVectorY > 0 )
503 nextReadAheadBlkY = ((int)paramsPtr->
m_blockY) + 1;
504 --globalReadAheadDirectionVectorY;
508 nextReadAheadBlkY = ((int)paramsPtr->
m_blockY) - 1;
509 ++globalReadAheadDirectionVectorY;
512 if( ( nextReadAheadBlkY >= 0 ) &&
513 ( nextReadAheadBlkY <
517 nextReadAheadBlkY = ((int)paramsPtr->
m_blockY);
522 globalReadAheadDirectionVectorX = 0;
523 globalReadAheadDirectionVectorY = 0;
524 globalReadAheadDirectionVectorB = 0;
532 assert( nextReadAheadBlkX <
534 assert( nextReadAheadBlkY <
538 nextReadAheadBlkY, internalExchangeBlockPtr );
540 globalReadAheadBlockX = nextReadAheadBlkX;
541 globalReadAheadBlockY = nextReadAheadBlkY;
542 globalReadAheadBlockB = nextReadAheadBlkB;
543 globalReadAheadBlockReady =
true;
549 lastReadBlockX = (int)paramsPtr->
m_blockX;
550 lastReadBlockY = (
int)paramsPtr->
m_blockY;
551 lastReadBlockB = (int)paramsPtr->
m_blockB;
553 else if( paramsPtr->
m_task == ThreadParameters::WriteTaskT )
566 paramsPtr->
m_task = ThreadParameters::InvalidTaskT;
569 else if( paramsPtr->
m_task == ThreadParameters::SuicideTastT )
boost::mutex m_taskFinishedMutex
used by the thread to inform a task finishment.
unsigned int m_globalBlocksNumberX
The maximum number of blocks (X axis) for all bands.
Internal read/write thread execution parameters.
void * getBlockPointer(unsigned int band, unsigned int x, unsigned int y)
Returns a pointer to the required data block.
unsigned int m_globalBlockSizeBytes
The maximum block size for all bands.
bool m_taskFinished
true when the thread has finished the required task.
int m_nblocksx
The number of blocks in x.
virtual const Band * getBand(std::size_t i) const =0
Returns the raster i-th band.
unsigned int m_b
Block band index.
int m_nblocksy
The number of blocks in y.
unsigned int m_dataPrefetchThreshold
The user defined read-ahead threshold.
boost::scoped_array< unsigned char > m_threadDataBlockHandler
A extra block used in exchange when a read-ahead task is performed.
unsigned int m_x
Block index over the X axis.
int getBlockSize() const
It returns the number of bytes ocuppied by a data block.
unsigned int m_y
Block index over the Y axis.
void initState()
Initialize this instance to an initial state.
RAM cached and tiled raster band blocks manager.
unsigned int m_blockB
Raster block band index.
CachedBandBlocksManager()
unsigned int m_blockY
Raster block Y index.
void read(int x, int y, void *buffer) const
It reads a data block to the specified buffer.
te::rst::Raster * m_rasterPtr
External raster pointer.
TECOMMONEXPORT unsigned long int GetUsedVirtualMemory()
Returns the amount of used virtual memory (bytes) for the current process (physical + swapped)...
An abstract class for raster data strucutures.
BandProperty * getProperty()
Returns the band property.
virtual std::size_t getNumberOfBands() const =0
Returns the number of bands (dimension of cells attribute values) in the raster.
unsigned int m_maxNumberOfCacheBlocks
The maximum number of cache blocks.
bool initialize(const te::rst::Raster &externalRaster, const unsigned char maxMemPercentUsed, const unsigned int dataPrefetchThreshold)
Initialize this instance to an initial state.
unsigned int m_globalBlocksNumberY
The maximum number of blocks (Y axis) for all bands.
te::rst::Raster * m_rasterPtr
External raster pointer.
unsigned int m_blockX
Raster block X index.
unsigned int m_dataPrefetchThreshold
The read-ahead data prefetch threshold (0-will disable prefetch, 1-data always prefetched, higher values will do prefetch when necessary).
unsigned int m_blocksFifoNextSwapBlockIndex
The next block swapp index over m_blocksFifo.
static void threadEntry(ThreadParameters *paramsPtr)
Thread entry.
unsigned char * m_blockPtr
Input block pointer.
unsigned char * m_getBlockPointer_BlkPtr
unsigned char * m_exchangeBlockPtr
Exchange block pointer.
boost::condition_variable m_taskFinishedCondVar
Used to wait for the required task finishment.
TECOMMONEXPORT unsigned long int GetTotalPhysicalMemory()
Returns the amount of total physical memory (bytes).
~CachedBandBlocksManager()
void write(int x, int y, void *buffer)
It writes a data block from the specified buffer.
TaskType m_task
The required task to be performed (read/write/exit).
boost::mutex m_doTaskMutex
Used when wakenning the thread to perform some task.
boost::condition_variable m_doTaskCondVar
Used by the thread when awakenning to perform some task.
TECOMMONEXPORT unsigned long int GetTotalVirtualMemory()
Returns the amount of total virtual memory (bytes) that can be claimed by the current process (physic...