diff options
| -rwxr-xr-x | fusearchive.py | 50 |
1 files changed, 40 insertions, 10 deletions
diff --git a/fusearchive.py b/fusearchive.py index 891872a..d4e8ebe 100755 --- a/fusearchive.py +++ b/fusearchive.py @@ -28,6 +28,11 @@ magic_blocksize = 1024 * 32 magic_depth = 5 debug_level = 0 +# Memory for dirty blocks, per file (1M) +dirty_size = 1024 * 1024 * 1; +# This is the number of actualy blocks in that size +dirty_flush = int( dirty_size / magic_blocksize ) + def dmsg(level,message): if level <= debug_level: print str(level) + ": " + str(message) @@ -278,10 +283,12 @@ class FuseArchive(Fuse): self.modified = False # This is the current in-memory chunk and offset in to data[] + self.chunk_cache = {}; self.chunk = '' self.chunk_index = 0 self.chunk_modified = False self.chunk_size = magic_blocksize + self.dirty_chunks = 0 # The chunk table self.chunks = [] @@ -339,10 +346,11 @@ class FuseArchive(Fuse): dmsg( 3, "Loading chunk " + str(index) ) key = None - try: - key = self.chunks[ index ] - except IndexError: + size = len( self.chunks ) + if index >= size: dmsg( 3, "Index doesn't exist" ) + else: + key = self.chunks[ index ] if key: dmsg( 3, "Index: " + str( key ) ) @@ -354,18 +362,37 @@ class FuseArchive(Fuse): self.chunk_index = index self.chunk_modified = False + # This simply puts the chunk data inside our current chunks at chunk_index def _save_chunk(self): if self.chunk_modified: - dmsg( 3, "Saving chunk " + str( self.chunk_index ) ) - key = save_chunk( self.chunk ) - # Make sure we have room for this chunk size = len( self.chunks ) if self.chunk_index >= size: - self.chunks.extend( [None] * ( self.chunk_index -size + 1 ) ) - - self.chunks[ self.chunk_index ] = key - dmsg( 3, "Key was " + str( key ) ) + self.chunks.extend( [ '' ] * ( self.chunk_index -size + 1 ) ) + + # Increment dirty chunks if we had a key here already + if isinstance( self.chunks[ self.chunk_index ], list ) or \ + len( self.chunks[ self.chunk_index ] ) == 0: + self.dirty_chunks += 1 + dmsg( 3, "Dirty chunks is now: " + str( self.dirty_chunks ) ) + dmsg( 3, "Dirty flush at: " + str( dirty_flush ) ) + + # Save the dirty chunk temporarily in memory + self.chunks[ self.chunk_index ] = self.chunk + + # Flush if we have too many dirty chunks + if self.dirty_chunks > dirty_flush: + self._flush_chunks() + + # This flushes any cached chunks + def _flush_chunks(self): + for index in range( len( self.chunks ) ): + if isinstance( self.chunks[ index ], str ): + dmsg( 3, "Flushing chunk at " + str( index ) ) + key = save_chunk( self.chunks[ index ] ) + self.chunks[ index ] = key + dmsg( 3, "Key was " + str( key ) ) + self.dirty_chunks = 0 def read(self, length, offset): dmsg( 3, "Reading from " + self.orig_path + " offset: " + str( offset ) @@ -454,6 +481,9 @@ class FuseArchive(Fuse): # Save our main data self._save_chunk() + # And flush any cached chunks + self._flush_chunks() + # Figure out our size based on the number of chunks + the # len of the final chunk numchunks = len( self.chunks ) |