aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorSteve Slaven <bpk@hoopajoo.net>2009-08-04 05:15:54 (GMT)
committerSteve Slaven <bpk@hoopajoo.net>2009-08-04 05:15:54 (GMT)
commit1af5ec1e05c0cd5dfa1737d0822aaaa73a5a9833 (patch)
tree58898a21a482b392928c7d1f092f60954bfaca64
parent2d3e545f3c7fc1c0e0ae2a78593199e4b76a04a6 (diff)
downloadfusearchive-1af5ec1e05c0cd5dfa1737d0822aaaa73a5a9833.zip
fusearchive-1af5ec1e05c0cd5dfa1737d0822aaaa73a5a9833.tar.gz
fusearchive-1af5ec1e05c0cd5dfa1737d0822aaaa73a5a9833.tar.bz2
Added binary format chunk serializer
-rw-r--r--FuseArchive/Chunk.py72
-rw-r--r--FuseArchive/Storage/FileSystem.py3
2 files changed, 74 insertions, 1 deletions
diff --git a/FuseArchive/Chunk.py b/FuseArchive/Chunk.py
new file mode 100644
index 0000000..c011382
--- /dev/null
+++ b/FuseArchive/Chunk.py
@@ -0,0 +1,72 @@
+import struct, zlib
+
+# Format version
+# Payload block size (so you don't have to uncompress it to see)
+# Data compression type (0 = none, 1 = zlib)
+# Ref count (number of files that use this)
+# filler to pad out ot 64 bytes for future expansion
+hformat = 'HHBL52x'
+compress_level = 6
+
+# This handles serialization and deserialization of compressed chunks with
+# some header data
+class Chunk:
+ def __init__(self):
+ self.chunk = ''
+ self.count = 0
+
+ #####
+ # Need a way to serialize/deserialize just headers? so we can update
+ # just counts, etc?
+ # Count needs to be tracked when we unlink?
+ #####
+
+ # Returns a string representing the serialized class to be sent to a
+ # file
+ def serialize(self, compression = 1):
+ data = struct.pack( hformat,
+ 0,
+ len( self.chunk ),
+ compression,
+ self.count
+ )
+
+ if compression == 0:
+ data += self.chunk
+ elif compression == 1:
+ data += zlib.compress( self.chunk, compress_level )
+ else:
+ raise ValueError( "Invalid compression type: %d" % compression )
+
+ return data
+
+ # Converts the output of serialize back to a chunk object
+ @staticmethod
+ def unserialize(data):
+ hd = Chunk.parse_header( data[ :64 ] )
+ obj = Chunk()
+ obj.count = hd[ 'count' ]
+
+ compression = hd[ 'compression' ]
+ if compression == 0:
+ obj.chunk = data[ 64: ]
+ elif compression == 1:
+ obj.chunk = zlib.decompress( data[64: ] )
+ else:
+ raise ValueError( "Invalid compression type: %d" % compression )
+
+ return obj
+
+ # Returns a dict of the header data, in case you don't want to
+ # unserialize the whole thing to see some attributes which would
+ # involve potentially uncompressing some data
+ @staticmethod
+ def parse_header(data):
+ fields = struct.unpack( hformat, data )
+ return {
+ 'version': fields[ 0 ],
+ 'size': fields[ 1 ],
+ 'compression': fields[ 2 ],
+ 'count': fields[ 3 ]
+ }
+
diff --git a/FuseArchive/Storage/FileSystem.py b/FuseArchive/Storage/FileSystem.py
index 4310481..547e567 100644
--- a/FuseArchive/Storage/FileSystem.py
+++ b/FuseArchive/Storage/FileSystem.py
@@ -1,5 +1,6 @@
import FuseArchive, logging, os, sha
from binascii import hexlify
+from FuseArchive.Chunk import Chunk
magic_depth = 5
@@ -51,7 +52,7 @@ def save_chunk( chunk ):
verify_contents = verify.read()
verify.close()
- verify_contents = FuseArchiveChunk.deserialize( verify_contents )
+ verify_contents = Chunk.deserialize( verify_contents )
if verify_contents.chunk == chunk:
logging.debug( "Found existing block" )
break