diff options
| -rwxr-xr-x | fusearchive.py | 148 |
1 files changed, 72 insertions, 76 deletions
diff --git a/fusearchive.py b/fusearchive.py index b95ec65..26e4571 100755 --- a/fusearchive.py +++ b/fusearchive.py @@ -9,10 +9,11 @@ # import os, sys, fcntl, fuse, sha, cPickle, gzip, errno -import zipfile, stat, logging +import zipfile, logging from fuse import Fuse +from binascii import hexlify -import pdb +#import pdb if not hasattr(fuse, '__version__'): raise RuntimeError, \ @@ -85,9 +86,9 @@ def _save_chunk_fs( chunk ): if magic_profiling: return( [ 0, 0 ] ) - logging.debug( "Begin save_chunk, length: " + str( len( chunk ) ) ) + logging.debug( "Begin save_chunk, length: %d" % len( chunk ) ) if deep_debug: - logging.debug( "Chunk: " + str( chunk ) ) + logging.debug( "Chunk: %s" + hexlify( chunk ) ) # Save this hash string, similar to the backuppc algo digest = sha.new( chunk ).digest() @@ -141,16 +142,16 @@ def _save_chunk_fs( chunk ): savechunk.close() break - logging.debug( "Got chunk slot: " + str( sub ) ) + logging.debug( "Got chunk slot: %d" % sub ) return( [ digest, sub ] ) def _save_chunk_zip( chunk ): if magic_profiling: return( [ 0, 0 ] ) - logging.debug( "Begin save_chunk, length: " + str( len( chunk ) ) ) + logging.debug( "Begin save_chunk, length: %d" % len( chunk ) ) if deep_debug: - logging.debug( "Chunk: " + str( chunk ) ) + logging.debug( "Chunk: %s" + hexlify( chunk ) ) # Save this hash string, similar to the backuppc algo digest = sha.new( chunk ).digest() @@ -201,7 +202,7 @@ def _save_chunk_zip( chunk ): break z.close() - logging.debug( "Got chunk slot: " + str( sub ) ) + logging.debug( "Got chunk slot: %d" % sub ) return( [ digest, sub ] ) # This will return a data block by key that was saved previously @@ -217,7 +218,7 @@ def _load_chunk_fs( key ): # Todo: make a digest -> path function to share with deflate hexdigest = ''.join( [ "%02x" % ord( x ) for x in chars ] ) - logging.debug( "Hash is: " + str(hexdigest) + " sub " + str(seq) ) + logging.debug( "Hash is: %s sub %d" % ( hexdigest, seq ) ) subparts = [ "%02x" % ord( x ) for x in chars[ :magic_depth ] ] subpath = '/'.join( subparts ) logging.debug( "Subpath: " + subpath ) @@ -235,7 +236,7 @@ def _load_chunk_fs( key ): raise IOError if deep_debug: - logging.debug( "Load-Chunk: " + str( chunk ) ) + logging.debug( "Load-Chunk: %s" + hexlify( chunk ) ) return chunk @@ -267,7 +268,7 @@ def _load_chunk_zip( key ): raise IOError if deep_debug: - logging.debug( "Load-Chunk: " + str( chunk ) ) + logging.debug( "Load-Chunk: %s" + hexlify( chunk ) ) z.close() return chunk @@ -291,7 +292,7 @@ class FuseArchiveSerializer: @staticmethod def dumpfh( fh, obj ): - logging.debug( "Going to serialize " + str( obj ) + " to " + str( fh ) ) + logging.debug( "Going to serialize %s to %s" % ( str( obj ), str( fh ) ) ) fh.truncate( 0 ) fh.seek( 0 ) f = gzip.GzipFile( None, "wb", gzip_compress_level, fh ) @@ -401,7 +402,7 @@ class FuseArchive(Fuse): def truncate(self, path, len): # Truncate using the ftruncate on the file - logging.debug( "Using FuseArchiveFile to truncate " + path + " to " + str(len) ) + logging.debug( "Using FuseArchiveFile to truncate %s to %d" % ( path, len) ) f = self.FuseArchiveFile( path, os.O_RDWR, 0 ) f.ftruncate(len) f.release( 0 ) @@ -551,7 +552,7 @@ class FuseArchive(Fuse): def _load_chunk( self, index ): # If the current chunk is the same as the chunk we're loading # just return - logging.debug( "_load_chunk: " + str( index ) ) + logging.debug( "_load_chunk: %d" % index ) if index == self.chunk_index: logging.debug( "Load chunk is same as current chunk, all done" ) @@ -560,7 +561,7 @@ class FuseArchive(Fuse): # Save this chunk if modified self._save_chunk() - logging.debug( "Loading chunk " + str(index) ) + logging.debug( "Loading chunk %d" % index ) key = None size = len( self.chunks ) @@ -574,13 +575,13 @@ class FuseArchive(Fuse): logging.debug( "Found cached dirty page" ) self.chunk = key else: - logging.debug( "Index: " + str( key ) ) + logging.debug( "Index: %s" % key ) self.chunk = load_chunk( key ) else: logging.debug( "No chunk at this index, loading nothing" ) self.chunk = '' - logging.debug( "Loaded chunk of length: " + str( len( self.chunk ) ) ) + logging.debug( "Loaded chunk of length: %d" % len( self.chunk ) ) self.chunk_index = index self.chunk_modified = False @@ -588,7 +589,7 @@ class FuseArchive(Fuse): # This simply puts the chunk data inside our current chunks at chunk_index def _save_chunk(self): if self.chunk_modified: - logging.debug( "Saving chunk " + str(self.chunk_index) ) + logging.debug( "Saving chunk %d" % self.chunk_index ) # Make sure we have room for this chunk size = len( self.chunks ) @@ -599,8 +600,8 @@ class FuseArchive(Fuse): if isinstance( self.chunks[ self.chunk_index ], list ) or \ len( self.chunks[ self.chunk_index ] ) == 0: self.dirty_chunks += 1 - logging.debug( "Dirty chunks is now: " + str( self.dirty_chunks ) ) - logging.debug( "Dirty flush at: " + str( dirty_flush ) ) + logging.debug( "Dirty chunks is now: %d" % self.dirty_chunks ) + logging.debug( "Dirty flush at: %d" % dirty_flush ) # Save the dirty chunk temporarily in memory self.chunks[ self.chunk_index ] = self.chunk @@ -613,17 +614,15 @@ class FuseArchive(Fuse): def _flush_chunks(self): for index in range( len( self.chunks ) ): if isinstance( self.chunks[ index ], str ): - logging.debug( "Flushing chunk at " + str( index ) ) + logging.debug( "Flushing chunk at %d" % index ) key = save_chunk( self.chunks[ index ] ) self.chunks[ index ] = key - logging.debug( "Key was " + str( key ) ) + logging.debug( "Key was %s" % key ) self.dirty_chunks = 0 def read(self, length, offset): - logging.debug( "Reading from " + self.orig_path + " offset: " + str( offset ) - + ' (0x%x) ' % offset - + " length: " + str( length ) - + ' (0x%x)' % length ) + logging.debug( "Reading from %s offset: %d (0x%x) length: %d (0x%d)" % + ( self.orig_path, offset, offset, length, length ) ) data_read = 0 data = '' @@ -633,7 +632,7 @@ class FuseArchive(Fuse): # Keep reading chunks until we have at least this much data while data_read < length and not is_eof: - logging.debug( "Pulling chunk data: " + str( index ) ) + logging.debug( "Pulling chunk data: %d" % index ) self._load_chunk( index ) if len(self.chunk): chunk_remaining = len(self.chunk) - rest @@ -642,11 +641,11 @@ class FuseArchive(Fuse): if data_left < chunk_remaining: to_read = data_left - logging.debug( "chunk_remaining: " + str( chunk_remaining ) ) - logging.debug( "data_left: " + str( data_left ) ) - logging.debug( "data_read: " + str( data_read ) ) - logging.debug( "rest: " + str( rest ) ) - logging.debug( "Copying " + str(to_read) + " bytes" ) + logging.debug( "chunk_remaining: %d" % chunk_remaining ) + logging.debug( "data_left: %d" % data_left ) + logging.debug( "data_read: %d" % data_read ) + logging.debug( "rest: %d" % rest ) + logging.debug( "Copying %d bytes" % to_read ) data += self.chunk[ rest:(rest+to_read) ] data_read += to_read @@ -656,38 +655,36 @@ class FuseArchive(Fuse): logging.debug( "No more chunk data, bye" ) is_eof = True - logging.debug( "Returning " + str( len( data ) ) + " bytes of data" ) - logging.debug( "Internal count was: " + str( data_read ) ) + logging.debug( "Returning %d bytes of data" % len( data ) ) + logging.debug( "Internal count was: %d" % data_read ) return data def write(self, buf, offset): if magic_profiling: return len( buf ) - logging.debug( "Writing to " + self.orig_path + " offset: " + str( offset ) + - ' (0x%x) ' % offset - + ' length: ' + str( len( buf ) ) - + ' (0x%x)' % len( buf ) ) + logging.debug( "Writing to %s offset: %d (0x%x) length: %d (0x%x)" % + ( self.orig_path, offset, offset, len( buf ), len( buf ) ) ) index = int( offset / self.chunk_size ) rest = offset % self.chunk_size - logging.debug( "This chunk falls on index: " + str( index ) + " rest: " + str( rest ) ) - logging.debug( "We have " + str( len( self.chunks ) ) + " chunks" ) - logging.debug( "File size is: " + str( self.size ) ) + logging.debug( "This chunk falls on index: %d rest: %d" % ( index, rest ) ) + logging.debug( "We have %d chunks" % len( self.chunks ) ) + logging.debug( "File size is: %d" % self.size ) # If index is higher than the number of blocks we current have it's a seek hole, so we need to extend our blocks out # We know these need to essentially be zeroed up to this size since if len( self.chunks ) - 1 < index: - logging.debug( "Not enough chunks " + str( len( self.chunks ) ) + ", need " + - str( index + 1 ) + ", extending" ) + logging.debug( "Not enough chunks %d, need %d, extending" % + ( len( self.chunks ), index + 1 ) ) this_index = 0 while this_index < index: self._load_chunk( this_index ) fill_null = self.chunk_size - len(self.chunk) - logging.debug( "Filling this chunk with null, bytes: " + str( fill_null ) ) + logging.debug( "Filling this chunk with null, bytes: %d" % fill_null ) self.chunk += "\0" * fill_null - logging.debug( "Chunk is now: " + str( len( self.chunk) ) + " bytes" ) + logging.debug( "Chunk is now: %d bytes" % len( self.chunk) ) self.chunk_modified = True self._save_chunk() this_index += 1 @@ -697,7 +694,7 @@ class FuseArchive(Fuse): # Now check if this chunk needs to be extended if len( self.chunk ) < rest: fill_null = rest - len(self.chunk) - logging.debug( "Filling final chunk with null, bytes: " + str( fill_null ) ) + logging.debug( "Filling final chunk with null, bytes: %d" % fill_null ) self.chunk += "\0" * fill_null self.chunk_modified = True self._save_chunk() @@ -705,31 +702,30 @@ class FuseArchive(Fuse): buf_offset = 0 buf_len = len(buf) - logging.debug( "Length: " + str( buf_len ) ) + logging.debug( "Length: %d" % buf_len ) while( buf_offset < buf_len ): - logging.debug( "Pulling in chunk for writing: " + str(index) ) + logging.debug( "Pulling in chunk for writing: %d" % index ) self._load_chunk( index ) buf_remain = buf_len - buf_offset chunk_remain = self.chunk_size - rest - logging.debug( "buf_remain: " + str(buf_remain) ) - logging.debug( "chunk_remain: " + str(chunk_remain) ) + logging.debug( "buf_remain: %d" % buf_remain ) + logging.debug( "chunk_remain: %d" % chunk_remain ) if chunk_remain < buf_remain: - logging.debug( "Writing " + str( chunk_remain ) + " bytes, buffer boundry" ) + logging.debug( "Writing %d bytes, buffer boundry" % chunk_remain ) this_len = chunk_remain else: - logging.debug( "Writing final " + str( buf_remain ) + " bytes" ) + logging.debug( "Writing final %d bytes" % buf_remain ) this_len = buf_remain - logging.debug( "Bytes to copy: " + str(this_len) ) - logging.debug( " buf offset: " + str(buf_offset) ) - logging.debug( " chunk offset: " + str(rest) ) - + logging.debug( "Bytes to copy: %d" % this_len ) + logging.debug( " buf offset: %d" % buf_offset ) + logging.debug( " chunk offset: %d" % rest ) if deep_debug: - logging.debug( "Pre-Buf: " + str(buf) ) - logging.debug( "Pre-Chunk: " + str(self.chunk) ) + logging.debug( "Pre-Buf: %s" % hexlify(buf) ) + logging.debug( "Pre-Chunk: %s" % hexlify(self.chunk) ) # Since python doesn't do in-place reassignment like you # can with splice() we will reconstruct the data by joining @@ -740,8 +736,8 @@ class FuseArchive(Fuse): self.chunk[ (rest + this_len): ] if deep_debug: - logging.debug( "Post-Buf: " + str(buf) ) - logging.debug( "Post-Chunk: " + str(self.chunk) ) + logging.debug( "Post-Buf: %s" % hexlify(buf) ) + logging.debug( "Post-Chunk: %s" % hexlify(self.chunk) ) buf_offset += this_len @@ -755,9 +751,9 @@ class FuseArchive(Fuse): if offset + len(buf) > self.size: self.size = offset + len(buf) - logging.debug( "This chunk size is now: " + str(len(self.chunk)) ) - logging.debug( "File size is now: " + str(self.size) ) - logging.debug( "Num Chunks: " + str( len( self.chunks ) ) ) + logging.debug( "This chunk size is now: %d" % len( self.chunk ) ) + logging.debug( "File size is now: %d" % self.size ) + logging.debug( "Num Chunks: %d" % len( self.chunks ) ) # Mark us in the dirty cache dirty_cache[ self.orig_path ] = self @@ -789,7 +785,7 @@ class FuseArchive(Fuse): numchunks = len( self.chunks ) if numchunks > 0: # Load the last chunk - logging.debug( "We have " + str(numchunks) + " chunks, calculating size" ) + logging.debug( "We have %d chunks, calculating size" % numchunks ) self._load_chunk( numchunks - 1 ) self.size = ( numchunks - 1 ) * self.chunk_size + \ len( self.chunk ) @@ -803,7 +799,7 @@ class FuseArchive(Fuse): + self.orig_path + " = " + str( self.size ) \ + " doesn't match internal size " + str( save_size ) \ + "\nProbably a bug in write or ftruncate!" - logging.debug( "Size calculated is: " + str( self.size ) + ' (0x%x)' % self.size ) + logging.debug( "Size calculated is: %d (0x%x)" % ( self.size, self.size ) ) FuseArchiveSerializer.dumpfh( self.file, { 'size': self.size, @@ -855,11 +851,11 @@ class FuseArchive(Fuse): curr_chunks = len( self.chunks ) need_chunks = ( length / self.chunk_size ) extra_bytes = length % self.chunk_size - logging.debug( "Ftruncate - " + str(length) + ' (0x%x) ' % length ) - logging.debug( " - self.size: " + str( self.size ) ) - logging.debug( " - curr_chunks: " + str( curr_chunks ) ) - logging.debug( " - need_chunks: " + str( need_chunks ) ) - logging.debug( " - extra_bytes: " + str( extra_bytes ) ) + logging.debug( "Ftruncate - %d (0x%x)" % ( length, length ) ) + logging.debug( " - self.size: %d" % self.size ) + logging.debug( " - curr_chunks: %d" % curr_chunks ) + logging.debug( " - need_chunks: %d" % need_chunks ) + logging.debug( " - extra_bytes: %d" % extra_bytes ) if extra_bytes: logging.debug( "Need an extra chunk" ) @@ -872,18 +868,18 @@ class FuseArchive(Fuse): self.chunks = [] self.chunk = '' elif self.size <= length: - logging.debug( "Need to pad out file, writing/seeking to " + str( length ) ) + logging.debug( "Need to pad out file, writing/seeking to %d" % length ) # Just write out null bytes to the length requested, write will do this for us if we specify the offset self.write( '', length ) else: logging.debug( "Truncating chunks" ) while True: - logging.debug( "Need chunks: " + str( need_chunks ) + " curr: " + str( curr_chunks ) ) + logging.debug( "Need chunks: %d curr: %d" % ( need_chunks, curr_chunks ) ) if need_chunks == curr_chunks: break - logging.debug( "Deleting chunk " + str( self.chunk_index ) ) + logging.debug( "Deleting chunk %d" % self.chunk_index ) self.chunks.pop() curr_chunks = len( self.chunks ) @@ -891,13 +887,13 @@ class FuseArchive(Fuse): # last chunk if len( self.chunks ): self._load_chunk( len( self.chunks ) - 1 ) - logging.debug( "Loaded final chunk, len: " + str( len( self.chunk ) ) ) + logging.debug( "Loaded final chunk, len: %d" % len( self.chunk ) ) # Now truncate this item if needed if len( self.chunk ) > extra_bytes: - logging.debug( "Truncating final chunk to " + str( extra_bytes ) ) + logging.debug( "Truncating final chunk to %d" % extra_bytes ) self.chunk = self.chunk[ :extra_bytes ] - logging.debug( "Chunk is now: " + str( len( self.chunk) ) + " bytes" ) + logging.debug( "Chunk is now: %d bytes" % len( self.chunk ) ) self.chunk_modified = True self.modified = True |