Merge pull request #24 from tk3369/tk/numarrayfn

Implementation of number_array_fn

Author: tk3369

src/SASLib.jl

@@ -28,6 +28,7 @@ struct ReaderConfig
     include_columns::Vector
     exclude_columns::Vector
     string_array_fn::Dict{Symbol, Function}
+    number_array_fn::Dict{Symbol, Function}
     verbose_level::Int64
 end
 
@@ -152,6 +153,7 @@ open(filename::AbstractString;
         include_columns::Vector = [],
         exclude_columns::Vector = [],
         string_array_fn::Dict = Dict(),
+        number_array_fn::Dict = Dict(),
         verbose_level::Int64 = 1)
 
 Open a SAS7BDAT data file.  Returns a `SASLib.Handler` object that can be used in
@@ -163,9 +165,10 @@ function open(filename::AbstractString;
         include_columns::Vector = [],
         exclude_columns::Vector = [],
         string_array_fn::Dict = Dict(),
+        number_array_fn::Dict = Dict(),
         verbose_level::Int64 = 1)
     return _open(ReaderConfig(filename, encoding, default_chunk_size, convert_dates, 
-        include_columns, exclude_columns, string_array_fn, verbose_level))
+        include_columns, exclude_columns, string_array_fn, number_array_fn, verbose_level))
 end
 
 """
@@ -205,6 +208,7 @@ readsas(filename::AbstractString;
         include_columns::Vector = [],
         exclude_columns::Vector = [],
         string_array_fn::Dict = Dict(),
+        number_array_fn::Dict = Dict(),
         verbose_level::Int64 = 1)
 
 Read a SAS7BDAT file.  
@@ -239,6 +243,10 @@ For examples,
 or  
 `string_array_fn = Dict(:column1 => REGULAR_STR_ARRAY)`.
 
+For numeric columns, you may specify your own array constructors using
+the `number_array_fn` parameter.  Perhaps you have a different kind of
+array to store the values e.g. SharedArray.
+
 For debugging purpose, `verbose_level` may be set to a value higher than 1.
 Verbose level 0 will output nothing to the console, essentially a total quiet 
 option.
@@ -249,11 +257,12 @@ function readsas(filename::AbstractString;
         include_columns::Vector = [],
         exclude_columns::Vector = [],
         string_array_fn::Dict = Dict(),
+        number_array_fn::Dict = Dict(),
         verbose_level::Int64 = 1)
     handler = nothing
     try
         handler = _open(ReaderConfig(filename, encoding, default_chunk_size, convert_dates, 
-            include_columns, exclude_columns, string_array_fn, verbose_level))
+            include_columns, exclude_columns, string_array_fn, number_array_fn, verbose_level))
         return read(handler)
     finally
         isdefined(handler, :string_decoder) && Base.close(handler.string_decoder)
@@ -982,7 +991,7 @@ function read_chunk(handler, nrows=0)
     perf_read_data = toq()
 
     tic()
-    rslt = _chunk_to_dataframe(handler)
+    rslt = _chunk_to_dataframe(handler, nrows)
     perf_chunk_to_data_frame = toq()
 
     # here column symbols contains only ones for columns that are actually read
@@ -1015,7 +1024,8 @@ function read_chunk(handler, nrows=0)
         :column_info => column_info,
         :compression => compressionstring(handler),
         :perf_read_data => perf_read_data,
-        :perf_type_conversion => perf_chunk_to_data_frame
+        :perf_type_conversion => perf_chunk_to_data_frame,
+        :process_id => myid()
         )
 end
 
@@ -1039,6 +1049,17 @@ function createstrarray(handler, column_symbol, nrows)
     end
 end
 
+# create numeric array
+function createnumarray(handler, column_symbol, nrows)
+    if haskey(handler.config.number_array_fn, column_symbol)
+        handler.config.number_array_fn[column_symbol](nrows)
+    elseif haskey(handler.config.number_array_fn, :_all_)
+        handler.config.number_array_fn[:_all_](nrows)
+    else
+        zeros(Float64, nrows)
+    end
+end
+
 function nullresult(filename)
     Dict(
         :data => Dict(), 
@@ -1121,7 +1142,7 @@ end
 # Construct Dict object that holds the columns.
 # For date or datetime columns, convert from numeric value to Date/DateTime type column.
 # The resulting dictionary uses column symbols as the key.
-function _chunk_to_dataframe(handler)
+function _chunk_to_dataframe(handler, nrows)
     # println("IN: _chunk_to_dataframe")
 
     n = handler.current_row_in_chunk_index
@@ -1137,7 +1158,8 @@ function _chunk_to_dataframe(handler)
             #if j == 1  && length(bytes) < 100  #debug only
                 # println("  bytes=$bytes")
             #end
-            values = convertfloat64f(bytes, handler.file_endianness)
+            values = createnumarray(handler, name, nrows)
+            convertfloat64f!(values, bytes, handler.file_endianness)
             #println(length(bytes))
             #rslt[name] = bswap(rslt[name])
             rslt[name] = values
@@ -1149,6 +1171,7 @@ function _chunk_to_dataframe(handler)
                     rslt[name] = datetime_from_float(rslt[name])
                 end
             end
+
         elseif ty == column_type_string
             # println("  String: size=$(size(handler.string_chunk))")
             # println("  String: column $j, name $name, size=$(size(handler.string_chunk[js, :]))")
@@ -1714,6 +1737,7 @@ function Base.show(io::IO, h::Handler)
     println(io, "  page size:   $(h.page_length)")
     println(io, "  pages:       $(h.page_count)")
     println(io, "  rows:        $(h.row_count)")
+    println(io, "  cols:        $(h.column_count)")
 end
 
 

src/utils.jl

@@ -161,10 +161,11 @@ end
 # Version f.  Best one so far!
 # julia> @btime convertfloat64f(r, :LittleEndian)
 #   35.132 μs (2 allocations: 78.20 KiB)
-function convertfloat64f(bytes::Vector{UInt8}, endianess::Symbol) 
+# 
+# results will be updated directly in the provided array `r`
+function convertfloat64f!(r::AbstractVector{Float64}, bytes::Vector{UInt8}, endianess::Symbol) 
     L = length(bytes)
     n = div(L, 8)               # numbers to convert
-    r = zeros(Float64, n)       # results
     j = 1                       # result index
     @inbounds for i in 1:8:L
         if endianess == :LittleEndian

test/perf_results_0.4.2/py_jl_homimp_50.md

@@ -0,0 +1,78 @@
+# Julia/Python Performance Test Result
+
+## Summary
+
+Julia is ~10.7x faster than Python/Pandas
+
+## Test File
+
+Iterations: 50
+
+Filename|Size|Rows|Columns|Numeric Columns|String Columns
+--------|----|----|-------|---------------|--------------
+homimp.sas7bdat|1.2 MB|46641|6|1|5
+
+## Python
+```
+$ python -V
+Python 3.6.3 :: Anaconda custom (64-bit)
+
+$ python perf_test1.py data_AHS2013/homimp.sas7bdat 50
+Minimum: 0.2720 seconds
+Median:  0.3014 seconds
+Mean:    0.3140 seconds
+Maximum: 0.4728 seconds
+
+```
+
+## Julia (ObjectPool)
+```
+Julia Version 0.6.2
+Commit d386e40c17 (2017-12-13 18:08 UTC)
+Platform Info:
+  OS: macOS (x86_64-apple-darwin14.5.0)
+  CPU: Intel(R) Core(TM) i5-4258U CPU @ 2.40GHz
+  WORD_SIZE: 64
+  BLAS: libopenblas (USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell)
+  LAPACK: libopenblas64_
+  LIBM: libopenlibm
+  LLVM: libLLVM-3.9.1 (ORCJIT, haswell)
+
+BenchmarkTools.Trial: 
+  memory estimate:  20.56 MiB
+  allocs estimate:  513299
+  --------------
+  minimum time:     47.109 ms (0.00% GC)
+  median time:      56.312 ms (11.21% GC)
+  mean time:        57.920 ms (10.72% GC)
+  maximum time:     78.471 ms (9.23% GC)
+  --------------
+  samples:          50
+  evals/sample:     1
+```
+
+## Julia (Regular String Array)
+```
+Julia Version 0.6.2
+Commit d386e40c17 (2017-12-13 18:08 UTC)
+Platform Info:
+  OS: macOS (x86_64-apple-darwin14.5.0)
+  CPU: Intel(R) Core(TM) i5-4258U CPU @ 2.40GHz
+  WORD_SIZE: 64
+  BLAS: libopenblas (USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell)
+  LAPACK: libopenblas64_
+  LIBM: libopenlibm
+  LLVM: libLLVM-3.9.1 (ORCJIT, haswell)
+
+BenchmarkTools.Trial: 
+  memory estimate:  19.37 MiB
+  allocs estimate:  512178
+  --------------
+  minimum time:     25.528 ms (0.00% GC)
+  median time:      39.970 ms (33.88% GC)
+  mean time:        41.932 ms (35.10% GC)
+  maximum time:     113.933 ms (76.81% GC)
+  --------------
+  samples:          50
+  evals/sample:     1
+```

test/perf_results_0.4.2/py_jl_numeric_1000000_2_100.md

@@ -0,0 +1,52 @@
+# Julia/Python Performance Test Result
+
+## Summary
+
+Julia is ~11.8x faster than Python/Pandas
+
+## Test File
+
+Iterations: 100
+
+Filename|Size|Rows|Columns|Numeric Columns|String Columns
+--------|----|----|-------|---------------|--------------
+numeric_1000000_2.sas7bdat|16.3 MB|1000000|2|2|0
+
+## Python
+```
+$ python -V
+Python 3.6.3 :: Anaconda custom (64-bit)
+
+$ python perf_test1.py data_misc/numeric_1000000_2.sas7bdat 100
+Minimum: 1.7937 seconds
+Median:  1.8426 seconds
+Mean:    1.8485 seconds
+Maximum: 2.0821 seconds
+
+```
+
+## Julia
+```
+Julia Version 0.6.2
+Commit d386e40c17 (2017-12-13 18:08 UTC)
+Platform Info:
+  OS: macOS (x86_64-apple-darwin14.5.0)
+  CPU: Intel(R) Core(TM) i5-4258U CPU @ 2.40GHz
+  WORD_SIZE: 64
+  BLAS: libopenblas (USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell)
+  LAPACK: libopenblas64_
+  LIBM: libopenlibm
+  LLVM: libLLVM-3.9.1 (ORCJIT, haswell)
+
+BenchmarkTools.Trial: 
+  memory estimate:  153.16 MiB
+  allocs estimate:  1002737
+  --------------
+  minimum time:     152.629 ms (3.05% GC)
+  median time:      231.873 ms (35.79% GC)
+  mean time:        203.540 ms (23.47% GC)
+  maximum time:     257.027 ms (38.52% GC)
+  --------------
+  samples:          25
+  evals/sample:     1
+```

test/perf_results_0.4.2/py_jl_productsales_100.md

@@ -0,0 +1,78 @@
+# Julia/Python Performance Test Result
+
+## Summary
+
+Julia is ~21.1x faster than Python/Pandas
+
+## Test File
+
+Iterations: 100
+
+Filename|Size|Rows|Columns|Numeric Columns|String Columns
+--------|----|----|-------|---------------|--------------
+productsales.sas7bdat|148.5 kB|1440|10|5|5
+
+## Python
+```
+$ python -V
+Python 3.6.3 :: Anaconda custom (64-bit)
+
+$ python perf_test1.py data_pandas/productsales.sas7bdat 100
+Minimum: 0.0292 seconds
+Median:  0.0316 seconds
+Mean:    0.0325 seconds
+Maximum: 0.0572 seconds
+
+```
+
+## Julia (ObjectPool)
+```
+Julia Version 0.6.2
+Commit d386e40c17 (2017-12-13 18:08 UTC)
+Platform Info:
+  OS: macOS (x86_64-apple-darwin14.5.0)
+  CPU: Intel(R) Core(TM) i5-4258U CPU @ 2.40GHz
+  WORD_SIZE: 64
+  BLAS: libopenblas (USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell)
+  LAPACK: libopenblas64_
+  LIBM: libopenlibm
+  LLVM: libLLVM-3.9.1 (ORCJIT, haswell)
+
+BenchmarkTools.Trial: 
+  memory estimate:  1.07 MiB
+  allocs estimate:  18583
+  --------------
+  minimum time:     2.084 ms (0.00% GC)
+  median time:      2.188 ms (0.00% GC)
+  mean time:        2.408 ms (3.88% GC)
+  maximum time:     5.143 ms (47.78% GC)
+  --------------
+  samples:          100
+  evals/sample:     1
+```
+
+## Julia (Regular String Array)
+```
+Julia Version 0.6.2
+Commit d386e40c17 (2017-12-13 18:08 UTC)
+Platform Info:
+  OS: macOS (x86_64-apple-darwin14.5.0)
+  CPU: Intel(R) Core(TM) i5-4258U CPU @ 2.40GHz
+  WORD_SIZE: 64
+  BLAS: libopenblas (USE64BITINT DYNAMIC_ARCH NO_AFFINITY Haswell)
+  LAPACK: libopenblas64_
+  LIBM: libopenlibm
+  LLVM: libLLVM-3.9.1 (ORCJIT, haswell)
+
+BenchmarkTools.Trial: 
+  memory estimate:  1.05 MiB
+  allocs estimate:  18510
+  --------------
+  minimum time:     1.382 ms (0.00% GC)
+  median time:      1.430 ms (0.00% GC)
+  mean time:        1.608 ms (7.05% GC)
+  maximum time:     5.258 ms (65.43% GC)
+  --------------
+  samples:          100
+  evals/sample:     1
+```