Added caching some dirty blocks

1 files changed, 40 insertions, 10 deletions

diff --git a/fusearchive.py b/fusearchive.py
index 891872a..d4e8ebe 100755
--- a/fusearchive.py
+++ b/fusearchive.py
@@ -28,6 +28,11 @@ magic_blocksize = 1024 * 32
 magic_depth = 5
 debug_level = 0
 
+# Memory for dirty blocks, per file (1M)
+dirty_size = 1024 * 1024 * 1;
+# This is the number of actualy blocks in that size
+dirty_flush = int( dirty_size / magic_blocksize )
+
 def dmsg(level,message):
     if level <= debug_level:
         print str(level) + ": " + str(message)
@@ -278,10 +283,12 @@ class FuseArchive(Fuse):
             self.modified = False
 
             # This is the current in-memory chunk and offset in to data[]
+            self.chunk_cache = {};
             self.chunk = ''
             self.chunk_index = 0
             self.chunk_modified = False
             self.chunk_size = magic_blocksize
+            self.dirty_chunks = 0
 
             # The chunk table
             self.chunks = []
@@ -339,10 +346,11 @@ class FuseArchive(Fuse):
             dmsg( 3, "Loading chunk " + str(index) )
             key = None
 
-            try:
-                key = self.chunks[ index ]
-            except IndexError:
+            size = len( self.chunks )
+            if index >= size:
                 dmsg( 3, "Index doesn't exist" )
+            else:
+                key = self.chunks[ index ]
 
             if key:
                 dmsg( 3, "Index: " + str( key ) )
@@ -354,18 +362,37 @@ class FuseArchive(Fuse):
             self.chunk_index = index
             self.chunk_modified = False
 
+        # This simply puts the chunk data inside our current chunks at chunk_index
         def _save_chunk(self):
             if self.chunk_modified:
-                dmsg( 3, "Saving chunk " + str( self.chunk_index ) )
-                key = save_chunk( self.chunk )
-
                 # Make sure we have room for this chunk
                 size = len( self.chunks )
                 if self.chunk_index >= size:
-                    self.chunks.extend( [None] * ( self.chunk_index  -size + 1 ) )
-
-                self.chunks[ self.chunk_index ] = key
-                dmsg( 3, "Key was " + str( key ) )
+                    self.chunks.extend( [ '' ] * ( self.chunk_index  -size + 1 ) )
+
+                # Increment dirty chunks if we had a key here already
+                if isinstance( self.chunks[ self.chunk_index ], list ) or \
+                        len( self.chunks[ self.chunk_index ] ) == 0:
+                    self.dirty_chunks += 1
+                    dmsg( 3, "Dirty chunks is now: " + str( self.dirty_chunks ) )
+                    dmsg( 3, "Dirty flush at: " + str( dirty_flush ) )
+
+                # Save the dirty chunk temporarily in memory
+                self.chunks[ self.chunk_index ] = self.chunk
+
+                # Flush if we have too many dirty chunks
+                if self.dirty_chunks > dirty_flush:
+                    self._flush_chunks()
+
+        # This flushes any cached chunks
+        def _flush_chunks(self):
+            for index in range( len( self.chunks ) ):
+                if isinstance( self.chunks[ index ], str ):
+                    dmsg( 3, "Flushing chunk at " + str( index ) )
+                    key = save_chunk( self.chunks[ index ] )
+                    self.chunks[ index ] = key
+                    dmsg( 3, "Key was " + str( key ) )
+                    self.dirty_chunks = 0
 
         def read(self, length, offset):
             dmsg( 3, "Reading from " + self.orig_path + " offset: " + str( offset )
@@ -454,6 +481,9 @@ class FuseArchive(Fuse):
                 # Save our main data
                 self._save_chunk()
 
+                # And flush any cached chunks
+                self._flush_chunks()
+
                 # Figure out our size based on the number of chunks + the
                 # len of the final chunk
                 numchunks = len( self.chunks )