1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
|
import struct, zlib
# Format version
# Payload block size (so you don't have to uncompress it to see)
# Data compression type (0 = none, 1 = zlib)
# Ref count (number of files that use this)
# filler to pad out ot 64 bytes for future expansion
hformat = 'HHBL52x'
compress_level = 6
# This handles serialization and deserialization of compressed chunks with
# some header data
class Chunk:
def __init__(self):
self.chunk = ''
self.count = 0
#####
# Need a way to serialize/deserialize just headers? so we can update
# just counts, etc?
# Count needs to be tracked when we unlink?
#####
# Returns a string representing the serialized class to be sent to a
# file
def serialize(self, compression = 1):
data = struct.pack( hformat,
0,
len( self.chunk ),
compression,
self.count
)
if compression == 0:
data += self.chunk
elif compression == 1:
data += zlib.compress( self.chunk, compress_level )
else:
raise ValueError( "Invalid compression type: %d" % compression )
return data
# Converts the output of serialize back to a chunk object
@staticmethod
def deserialize(data):
hd = Chunk.parse_header( data[ :64 ] )
obj = Chunk()
obj.count = hd[ 'count' ]
compression = hd[ 'compression' ]
if compression == 0:
obj.chunk = data[ 64: ]
elif compression == 1:
obj.chunk = zlib.decompress( data[64: ] )
else:
raise ValueError( "Invalid compression type: %d" % compression )
return obj
# Returns a dict of the header data, in case you don't want to
# unserialize the whole thing to see some attributes which would
# involve potentially uncompressing some data
@staticmethod
def parse_header(data):
fields = struct.unpack( hformat, data )
return {
'version': fields[ 0 ],
'size': fields[ 1 ],
'compression': fields[ 2 ],
'count': fields[ 3 ]
}
|
