smarter compression

seung-lab · Dec 19, 2017 · a6344eb · a6344eb
1 parent e198eb0
commit a6344eb
Show file tree

Hide file tree

Showing 4 changed files with 18 additions and 13 deletions.
diff --git a/src/BigArrays.jl b/src/BigArrays.jl
@@ -31,7 +31,7 @@ const DATATYPE_MAP = Dict{String, DataType}(
 
 const CODING_MAP = Dict{String,Any}(
     # note that the raw encoding in cloud storage will be automatically encoded using gzip!
-    "raw"       => RawCoding,
+    "raw"       => GZipCoding,
     "jpeg"      => JPEGCoding,
     "blosclz"   => BlosclzCoding,
     "gzip"      => GZipCoding 

diff --git a/src/Codings.jl b/src/Codings.jl
@@ -9,6 +9,8 @@ abstract type AbstractBigArrayCoding end
 export AbstractBigArrayCoding, JPEGCoding, RawCoding, BlosclzCoding, GZipCoding
 export encode, decode 
 
+const GZIP_MAGIC_NUMBER = UInt8[0x1f, 0x8b, 0x08]
+
 function __init__()
     # use the same number of threads with Julia
     if haskey(ENV, "BLOSC_NUM_THREADS")
@@ -43,7 +45,11 @@ function encode(data::Array, coding::Type{GZipCoding})
 end
 
 function decode(data::Vector{UInt8}, coding::Type{GZipCoding})
-    Libz.inflate(data)
+    if all(data[1:3] .== GZIP_MAGIC_NUMBER)
+        return Libz.inflate(data)
+    else 
+        return data 
+    end
 end
 
 function encode( data::Array, coding::Type{BlosclzCoding} )

diff --git a/src/backends/BinDicts.jl b/src/backends/BinDicts.jl
@@ -1,5 +1,5 @@
 module BinDicts
-using Libz
+#using Libz
 
 import ..BackendBase: AbstractBigArrayBackend, get_info, get_scale_name 
 export BinDict, get_info, get_scale_name  
@@ -24,18 +24,17 @@ function get_scale_name( self::BinDict )
 end 
 
 function Base.getindex( self::BinDict, key::AbstractString)
-    f = open( joinpath( get_path(self), key ))
-    data = read(f)
-    close(f)
-    Libz.inflate(data)
+    open( joinpath( get_path(self), key )) do f
+        return read(f)
+        #Libz.inflate(data)
+    end
 end 
 
 function Base.setindex!( self::BinDict, value::Array, key::AbstractString )
-    data = Libz.deflate(reinterpret(UInt8, value[:]))
+    data = reinterpret(UInt8, value[:])
+    #data = Libz.deflate( data )
     fileName = joinpath( get_path(self), key )
-    f = open(fileName, "w")
-    write(f, data)
-    close(f)
+    write(fileName, data)
 end 
 
 end # module
diff --git a/test/BinDicts.jl b/test/BinDicts.jl
@@ -9,8 +9,8 @@ mkdir(tempDir)
 mkdir(datasetDir)
 infoString = """
 {"num_channels": 1, "type": "image", "data_type": "uint8", "scales": [
-{"encoding": "raw", "chunk_sizes": [[100, 100, 5]], "key": "6_6_30", "resolution": [6, 6, 30], "voxel_offset": [0, 0, 0], "size": [12286, 11262, 2046]}, 
-{"encoding": "raw", "chunk_sizes": [[100, 100, 5]], "key": "12_12_30", "resolution": [12, 12, 30], "voxel_offset": [103, 103, 7], "size": [12286, 11262, 2046]} 
+{"encoding": "gzip", "chunk_sizes": [[100, 100, 5]], "key": "6_6_30", "resolution": [6, 6, 30], "voxel_offset": [0, 0, 0], "size": [12286, 11262, 2046]}, 
+{"encoding": "gzip", "chunk_sizes": [[100, 100, 5]], "key": "12_12_30", "resolution": [12, 12, 30], "voxel_offset": [103, 103, 7], "size": [12286, 11262, 2046]} 
 ]} 
 """
 open( joinpath(tempDir, "info"), "w" ) do f