import FuseArchive, logging, os, sha from binascii import hexlify from FuseArchive.Chunk import Chunk magic_depth = 5 # This will write out a data block, it will return a key that can get this # data back later def save_chunk( chunk ): if FuseArchive.magic_profiling: return( [ 0, 0 ] ) logging.debug( "Begin save_chunk, length: %d" % len( chunk ) ) if FuseArchive.deep_debug: logging.debug( "Chunk: %s" + hexlify( chunk ) ) # Save this hash string, similar to the backuppc algo digest = sha.new( chunk ).digest() # Write out our chunk chars = list( digest ) logging.debug( chars ) # We make the hexdigest here, yeah we could just call hexdigest() # but we need to essentially do this same thing to reassemble the # file anyway hexdigest = ''.join( [ "%02x" % ord( x ) for x in chars ] ) # Subparts just needs the first N chars subparts = [ "%02x" % ord( x ) for x in chars[ :magic_depth ] ] logging.debug( subparts ) subpath = '/'.join( subparts ) logging.debug( "Subpath: " + subpath ) # Make sure this sub path exists nextpart = "./storage" for part in subparts: nextpart += "/" + part if not os.path.exists( nextpart ): logging.debug( "Creating subdir: " + nextpart ) os.mkdir( nextpart ) # Find a chunk slot sub = 0 while True: checkpath = "./storage/%s/%s_%d" % ( subpath, hexdigest, sub ) logging.debug( "Checking: " + checkpath ) if os.path.exists( checkpath ): # Check if this is our data verify = open( checkpath, "rb" ) verify_contents = verify.read() verify.close() verify_contents = Chunk.deserialize( verify_contents ) if verify_contents.chunk == chunk: logging.debug( "Found existing block" ) break else: logging.debug( "Block exists but is not the same" ) sub += 1 else: # We found a spot, dump our data here logging.debug( "No block here, creating new block" ) save_chunk = Chunk() save_chunk.chunk = chunk savechunk = open( checkpath, "wb" ) savechunk.write( save_chunk.serialize() ) savechunk.close() break logging.debug( "Got chunk slot: %d" % sub ) return( [ digest, sub ] ) # This will return a data block by key that was saved previously def load_chunk( key ): if FuseArchive.magic_profiling: return '' ( thash, seq ) = key logging.debug( "Begin load_chunk" ) chars = list( thash ) logging.debug( chars ) # Todo: make a digest -> path function to share with deflate hexdigest = ''.join( [ "%02x" % ord( x ) for x in chars ] ) logging.debug( "Hash is: %s sub %d" % ( hexdigest, seq ) ) subparts = [ "%02x" % ord( x ) for x in chars[ :magic_depth ] ] subpath = '/'.join( subparts ) logging.debug( "Subpath: " + subpath ) subpath += "/%s_%d" % ( hexdigest, seq ) logging.debug( "Chunk path: " + subpath ) if os.path.exists( "./storage/" + subpath ): logging.debug( "Exporting chunk" ) readchunk = open( "./storage/" + subpath ) chunk = Chunk.deserialize( readchunk.read() ).chunk readchunk.close() else: raise IOError if FuseArchive.deep_debug: logging.debug( "Load-Chunk: %s" + hexlify( chunk ) ) return chunk