import os, logging, FuseArchive, zipfile, sha from binascii import hexlify def save_chunk( chunk ): if FuseArchive.magic_profiling: return( [ 0, 0 ] ) logging.debug( "Begin save_chunk, length: %d" % len( chunk ) ) if FuseArchive.deep_debug: logging.debug( "Chunk: %s" + hexlify( chunk ) ) # Save this hash string, similar to the backuppc algo digest = sha.new( chunk ).digest() # Write out our chunk chars = list( digest ) logging.debug( chars ) # We make the hexdigest here, yeah we could just call hexdigest() # but we need to essentially do this same thing to reassemble the # file anyway hexdigest = ''.join( [ "%02x" % ord( x ) for x in chars ] ) # Should be about max of 32k zip files zipname = hexdigest[ 0:4 ] + ".zip" logging.debug( "Zip name: " + zipname ) if not os.path.exists( "./storage/" + zipname ): logging.debug( "Creating intial empty zip" ) z = zipfile.ZipFile( "./storage/" + zipname, 'w', zipfile.ZIP_DEFLATED, True ) # append mode throws an exception if it's not zip, or maybe it's # just zero-length files z.writestr( 'junk', 'junk' ) z.close() z = zipfile.ZipFile( "./storage/" + zipname, 'a', zipfile.ZIP_DEFLATED, True ) # Find a chunk slot sub = 0 while True: checkpath = "%s_%d" % ( hexdigest, sub ) logging.debug( "Checking: " + checkpath ) try: data = z.read( checkpath ) except: data = '' if len(data): if data == chunk: logging.debug( "Found existing block" ) break else: logging.debug( "Block exists but is not the same" ) sub += 1 else: # We found a spot, dump our data here logging.debug( "No block here, creating new block" ) z.writestr( checkpath, chunk ) break z.close() logging.debug( "Got chunk slot: %d" % sub ) return( [ digest, sub ] ) def load_chunk( key ): if FuseArchive.magic_profiling: return '' ( thash, seq ) = key logging.debug( "Begin load_chunk" ) chars = list( thash ) logging.debug( chars ) # Todo: make a digest -> path function to share with deflate hexdigest = ''.join( [ "%02x" % ord( x ) for x in chars ] ) zipname = hexdigest[ 0:4 ] + ".zip" logging.debug( "Zip name: " + zipname ) z = zipfile.ZipFile( "./storage/" + zipname, 'r', zipfile.ZIP_DEFLATED, True ) subpath = "%s_%d" % ( hexdigest, seq ) logging.debug( "Chunk path: " + subpath ) data = z.read( subpath ) if len( data ): logging.debug( "Exporting chunk" ) chunk = data else: z.close() raise IOError if FuseArchive.deep_debug: logging.debug( "Load-Chunk: %s" + hexlify( chunk ) ) z.close() return chunk