Switched to cpickle and added wrappers for file open/dump/load stuff so

it's easier to experiment with different methods

1 files changed, 32 insertions, 13 deletions

diff --git a/fusearchive.py b/fusearchive.py
index 0609124..53a2d09 100755
--- a/fusearchive.py
+++ b/fusearchive.py
@@ -9,12 +9,12 @@
 #
 
 import os, sys, shutil, fcntl, fuse, re
-import tempfile, sha, pickle, gzip
+import tempfile, sha, cPickle, gzip
 from errno import *
 from stat import *
 from fuse import Fuse
 
-import pdb
+#import pdb
 
 if not hasattr(fuse, '__version__'):
     raise RuntimeError, \
@@ -27,7 +27,7 @@ fuse.feature_assert('stateful_files', 'has_init')
 magic_blocksize = 1024 * 128
 magic_depth = 5
 debug_level = 0
-gzip_compress_level = 1
+gzip_compress_level = 6
 
 # Memory for dirty blocks, per file (1M)
 dirty_size = 1024 * 1024 * 1;
@@ -76,7 +76,7 @@ def save_chunk( chunk ):
         dmsg( 3, "Checking: " + checkpath )
         if os.path.exists( checkpath ):
             # Check if this is our data
-            verify = gzip.open( checkpath, "rb" )
+            verify = FuseArchiveWriter.open( checkpath, "rb" )
             verify_contents = verify.read()
             verify.close()
 
@@ -89,7 +89,7 @@ def save_chunk( chunk ):
         else:
             # We found a spot, dump our data here
             dmsg( 3, "No block here, creating new block" )
-            savechunk = gzip.open( checkpath, "wb", gzip_compress_level )
+            savechunk = FuseArchiveWriter.open( checkpath, "wb" )
             savechunk.write( chunk )
             savechunk.close
             break
@@ -118,7 +118,7 @@ def load_chunk( key ):
 
     if os.path.exists( "./storage/" + subpath ):
         dmsg( 3, "Exporting chunk" )
-        readchunk = gzip.open( "./storage/" + subpath )
+        readchunk = FuseArchiveWriter.open( "./storage/" + subpath )
         chunk = readchunk.read()
         readchunk.close()
     else:
@@ -126,6 +126,28 @@ def load_chunk( key ):
 
     return chunk
 
+class FuseArchiveWriter:
+    """This just allows switching out writer classes easily"""
+    @staticmethod
+    def open( path, mode = 'r' ):
+        #return gzip.open( path, mode, gzip_compress_level )
+        return open( path, mode )
+
+class FuseArchiveSerializer:
+    """This lets us experiment with different main file serializers"""
+    @staticmethod
+    def dump( file, obj ):
+        out = FuseArchiveWriter.open( file, "wb" )
+        cPickle.dump( obj, out, -1 ) # new file format
+        out.close()
+
+    @staticmethod
+    def load( file ):
+        inp = FuseArchiveWriter.open( file, "rb" )
+        magic = cPickle.load( inp )
+        inp.close()
+        return magic
+
 class FuseArchiveStat(fuse.Stat):
     def __init__(self, stat):
         self.st_mode = stat.st_mode
@@ -315,9 +337,7 @@ class FuseArchive(Fuse):
                 src = "./tree" + self.orig_path
                 dmsg( 3, "Unpickling: " + src )
                 # TODO: return an IO error if inflating fails
-                inp = gzip.open( src, "rb" )
-                magic = pickle.load( inp )
-                inp.close()
+                magic = FuseArchiveSerializer.load( src )
                 dmsg( 3, "Got data: " + str( magic ) )
                 self.size = magic[ 'size' ]
                 self.chunks = magic[ 'chunks' ]
@@ -504,13 +524,12 @@ class FuseArchive(Fuse):
 
                 dmsg( 3, "Size calculated is: " + str( self.size ) )
 
-                out = gzip.open( "./tree" + self.orig_path, "wb", gzip_compress_level )
-                pickle.dump( {
+                FuseArchiveSerializer.dump( "./tree" + self.orig_path, {
                     'size': self.size,
                     'chunks': self.chunks,
                     'chunk_size': self.chunk_size
-                }, out )
-                out.close()
+                } )
+
 
             return 1