diff options
| -rw-r--r-- | FuseArchive/Storage/FileSystem.py | 101 | ||||
| -rw-r--r-- | FuseArchive/Storage/ZipFile.py | 96 | ||||
| -rw-r--r-- | FuseArchive/Storage/__init__.py | 0 |
3 files changed, 197 insertions, 0 deletions
diff --git a/FuseArchive/Storage/FileSystem.py b/FuseArchive/Storage/FileSystem.py new file mode 100644 index 0000000..8488964 --- /dev/null +++ b/FuseArchive/Storage/FileSystem.py @@ -0,0 +1,101 @@ +# This will write out a data block, it will return a key that can get this +# data back later +def save_chunk( chunk ): + if magic_profiling: + return( [ 0, 0 ] ) + + logging.debug( "Begin save_chunk, length: %d" % len( chunk ) ) + if deep_debug: + logging.debug( "Chunk: %s" + hexlify( chunk ) ) + + # Save this hash string, similar to the backuppc algo + digest = sha.new( chunk ).digest() + + # Write out our chunk + chars = list( digest ) + logging.debug( chars ) + + # We make the hexdigest here, yeah we could just call hexdigest() + # but we need to essentially do this same thing to reassemble the + # file anyway + hexdigest = ''.join( [ "%02x" % ord( x ) for x in chars ] ) + + # Subparts just needs the first N chars + subparts = [ "%02x" % ord( x ) for x in chars[ :magic_depth ] ] + + logging.debug( subparts ) + subpath = '/'.join( subparts ) + logging.debug( "Subpath: " + subpath ) + + # Make sure this sub path exists + nextpart = "./storage" + for part in subparts: + nextpart += "/" + part + if not os.path.exists( nextpart ): + logging.debug( "Creating subdir: " + nextpart ) + os.mkdir( nextpart ) + + # Find a chunk slot + sub = 0 + while True: + checkpath = "./storage/%s/%s_%d" % ( subpath, hexdigest, sub ) + logging.debug( "Checking: " + checkpath ) + if os.path.exists( checkpath ): + # Check if this is our data + verify = open( checkpath, "rb" ) + verify_contents = verify.read() + verify.close() + + verify_contents = FuseArchiveChunk.deserialize( verify_contents ) + if verify_contents.chunk == chunk: + logging.debug( "Found existing block" ) + break + else: + logging.debug( "Block exists but is not the same" ) + sub += 1 + else: + # We found a spot, dump our data here + logging.debug( "No block here, creating new block" ) + savechunk = open( checkpath, "wb" ) + savechunk.write( chunk ) + savechunk.close() + break + + logging.debug( "Got chunk slot: %d" % sub ) + return( [ digest, sub ] ) + +# This will return a data block by key that was saved previously +def load_chunk( key ): + if magic_profiling: + return '' + + ( thash, seq ) = key + logging.debug( "Begin load_chunk" ) + + chars = list( thash ) + logging.debug( chars ) + + # Todo: make a digest -> path function to share with deflate + hexdigest = ''.join( [ "%02x" % ord( x ) for x in chars ] ) + logging.debug( "Hash is: %s sub %d" % ( hexdigest, seq ) ) + subparts = [ "%02x" % ord( x ) for x in chars[ :magic_depth ] ] + subpath = '/'.join( subparts ) + logging.debug( "Subpath: " + subpath ) + + subpath += "/%s_%d" % ( hexdigest, seq ) + + logging.debug( "Chunk path: " + subpath ) + + if os.path.exists( "./storage/" + subpath ): + logging.debug( "Exporting chunk" ) + readchunk = FuseArchiveStream.open( "./storage/" + subpath ) + chunk = readchunk.read() + readchunk.close() + else: + raise IOError + + if deep_debug: + logging.debug( "Load-Chunk: %s" + hexlify( chunk ) ) + + return chunk + diff --git a/FuseArchive/Storage/ZipFile.py b/FuseArchive/Storage/ZipFile.py new file mode 100644 index 0000000..c76b5bb --- /dev/null +++ b/FuseArchive/Storage/ZipFile.py @@ -0,0 +1,96 @@ +import os, logging + +def save_chunk( chunk ): + if FuseArchive.magic_profiling: + return( [ 0, 0 ] ) + + logging.debug( "Begin save_chunk, length: %d" % len( chunk ) ) + if deep_debug: + logging.debug( "Chunk: %s" + hexlify( chunk ) ) + + # Save this hash string, similar to the backuppc algo + digest = sha.new( chunk ).digest() + + # Write out our chunk + chars = list( digest ) + logging.debug( chars ) + + # We make the hexdigest here, yeah we could just call hexdigest() + # but we need to essentially do this same thing to reassemble the + # file anyway + hexdigest = ''.join( [ "%02x" % ord( x ) for x in chars ] ) + + # Should be about max of 32k zip files + zipname = hexdigest[ 0:4 ] + ".zip" + logging.debug( "Zip name: " + zipname ) + if not os.path.exists( "./storage/" + zipname ): + logging.debug( "Creating intial empty zip" ) + z = zipfile.ZipFile( "./storage/" + zipname, 'w', zipfile.ZIP_DEFLATED, True ) + # append mode throws an exception if it's not zip, or maybe it's + # just zero-length files + z.writestr( 'junk', 'junk' ) + z.close() + + z = zipfile.ZipFile( "./storage/" + zipname, 'a', zipfile.ZIP_DEFLATED, True ) + + # Find a chunk slot + sub = 0 + while True: + checkpath = "%s_%d" % ( hexdigest, sub ) + logging.debug( "Checking: " + checkpath ) + try: + data = z.read( checkpath ) + except: + data = '' + + if len(data): + if data == chunk: + logging.debug( "Found existing block" ) + break + else: + logging.debug( "Block exists but is not the same" ) + sub += 1 + else: + # We found a spot, dump our data here + logging.debug( "No block here, creating new block" ) + z.writestr( checkpath, chunk ) + break + + z.close() + logging.debug( "Got chunk slot: %d" % sub ) + return( [ digest, sub ] ) + +def load_chunk( key ): + if FuseArchive.magic_profiling: + return '' + + ( thash, seq ) = key + logging.debug( "Begin load_chunk" ) + + chars = list( thash ) + logging.debug( chars ) + + # Todo: make a digest -> path function to share with deflate + hexdigest = ''.join( [ "%02x" % ord( x ) for x in chars ] ) + + zipname = hexdigest[ 0:4 ] + ".zip" + logging.debug( "Zip name: " + zipname ) + z = zipfile.ZipFile( "./storage/" + zipname, 'r', zipfile.ZIP_DEFLATED, True ) + + subpath = "%s_%d" % ( hexdigest, seq ) + logging.debug( "Chunk path: " + subpath ) + data = z.read( subpath ) + if len( data ): + logging.debug( "Exporting chunk" ) + chunk = data + else: + z.close() + raise IOError + + if deep_debug: + logging.debug( "Load-Chunk: %s" + hexlify( chunk ) ) + + z.close() + return chunk + + diff --git a/FuseArchive/Storage/__init__.py b/FuseArchive/Storage/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/FuseArchive/Storage/__init__.py |