diff --git a/.clang-format b/.clang-format index a7899a5..c421b98 100644 --- a/.clang-format +++ b/.clang-format @@ -1,16 +1,42 @@ --- +# See https://releases.llvm.org/14.0.0/tools/clang/docs/ClangFormatStyleOptions.html BasedOnStyle: Chromium AccessModifierOffset: -4 +AlignAfterOpenBracket: AlwaysBreak +# AlignArrayOfStructures can cause crashes, see https://github.com/llvm/llvm-project/issues/55269 +#AlignArrayOfStructures: Left +AllowAllParametersOfDeclarationOnNextLine: false +AllowShortBlocksOnASingleLine: Empty AllowShortFunctionsOnASingleLine: All -BreakConstructorInitializers: BeforeColon +BinPackArguments: false +BinPackParameters: false +BreakBeforeBinaryOperators: NonAssignment +BreakBeforeBraces: Custom +BraceWrapping: + # NB: due to https://github.com/llvm/llvm-project/issues/55582 the Multiline setting will not + # always work (should be fixed in clang-format 15, but that is not available as a python wheel yet + # due to https://github.com/ssciwr/clang-format-wheel/issues/49) + AfterControlStatement: MultiLine # makes sure multiline ifs don't run into their bodies + AfterFunction: true # makes constructors with initialisers much nicer +BreakBeforeConceptDeclarations: true +BreakBeforeTernaryOperators: true +BreakConstructorInitializers: BeforeComma +BreakStringLiterals: true ColumnLimit: 100 -ConstructorInitializerIndentWidth: 8 -ContinuationIndentWidth: 8 +CompactNamespaces: true +ConstructorInitializerIndentWidth: 0 +ContinuationIndentWidth: 4 Cpp11BracedListStyle: true -DerivePointerAlignment: true +DerivePointerAlignment: false # force use of the PointerAlignment setting FixNamespaceComments: true IncludeBlocks : Regroup IncludeCategories: + # Aim is: + # 0. the "main" header file (#include "foo.h" in foo.cpp) automatically gets priority 0 + # 1. internal headers (#include "util/helpers.h"): quotation marks, with a '/' + # 2. third-party headers (#include ): angle brackets, '/' or .h/.hpp/.h++ + # file ext + # 3. standard library headers (#include ): angle brackets, no file ext, no '/' - Regex: '^"' Priority: 1 - Regex: '^<.*/' @@ -22,12 +48,22 @@ IncludeCategories: - Regex: '\.h\+\+>' Priority: 2 IncludeIsMainRegex: '(_test|_tests|Tests|Test)?$' + # foo.h will be considered the "main" header (and sorted to the top) for all of the following: + # - foo.cpp + # - foo_test.cpp + # - foo_tests.cpp + # - fooTests.cpp (although this is intended for Foo.h and FooTests.cpp) + # - fooTest.cpp (although this is intended for Foo.h and FooTest.cpp) IndentCaseLabels: false IndentWidth: 4 +PackConstructorInitializers: CurrentLine PointerAlignment: Middle +QualifierAlignment: Right # const east +# clang 14 *should* know about QualifierOrder (according to its docs) but claims it doesn't +#QualifierOrder: ['static', 'constexpr', 'inline', 'type', 'const', 'volatile', 'restrict'] ReflowComments: false -SortIncludes: true +SeparateDefinitionBlocks: Always +SortIncludes: CaseInsensitive SortUsingDeclarations: true -Standard: Cpp11 - -... +SpaceAroundPointerQualifiers: Before +Standard: c++20 diff --git a/c++/examples/find_all_read_ids.cpp b/c++/examples/find_all_read_ids.cpp index 6868216..d8288c6 100644 --- a/c++/examples/find_all_read_ids.cpp +++ b/c++/examples/find_all_read_ids.cpp @@ -7,7 +7,8 @@ #include #include -int main(int argc, char** argv) { +int main(int argc, char ** argv) +{ if (argc != 2) { std::cerr << "Expected one argument - an pod5 file to search\n"; } @@ -16,7 +17,7 @@ int main(int argc, char** argv) { pod5_init(); // Open the file ready for walking: - Pod5FileReader_t* file = pod5_open_file(argv[1]); + Pod5FileReader_t * file = pod5_open_file(argv[1]); if (!file) { std::cerr << "Failed to open file " << argv[1] << ": " << pod5_get_error_string() << "\n"; return EXIT_FAILURE; @@ -35,7 +36,7 @@ int main(int argc, char** argv) { std::size_t read_count = 0; for (std::size_t batch_index = 0; batch_index < batch_count; ++batch_index) { - Pod5ReadRecordBatch_t* batch = nullptr; + Pod5ReadRecordBatch_t * batch = nullptr; if (pod5_get_read_batch(&batch, file, batch_index) != POD5_OK) { std::cerr << "Failed to get batch: " << pod5_get_error_string() << "\n"; return EXIT_FAILURE; @@ -50,8 +51,10 @@ int main(int argc, char** argv) { for (std::size_t row = 0; row < batch_row_count; ++row) { uint16_t read_table_version = 0; ReadBatchRowInfo_t read_data; - if (pod5_get_read_batch_row_info_data(batch, row, READ_BATCH_ROW_INFO_VERSION, - &read_data, &read_table_version) != POD5_OK) { + if (pod5_get_read_batch_row_info_data( + batch, row, READ_BATCH_ROW_INFO_VERSION, &read_data, &read_table_version) + != POD5_OK) + { std::cerr << "Failed to get read " << row << "\n"; return EXIT_FAILURE; } diff --git a/c++/examples/find_specific_read_ids.cpp b/c++/examples/find_specific_read_ids.cpp index 340c347..e994cde 100644 --- a/c++/examples/find_specific_read_ids.cpp +++ b/c++/examples/find_specific_read_ids.cpp @@ -8,7 +8,8 @@ #include #include -int main(int argc, char** argv) { +int main(int argc, char ** argv) +{ if (argc != 3) { std::cerr << "Expected two arguments:\n" << " - an pod5 file to search\n" @@ -19,7 +20,7 @@ int main(int argc, char** argv) { pod5_init(); // Open the file ready for walking: - Pod5FileReader_t* file = pod5_open_file(argv[1]); + Pod5FileReader_t * file = pod5_open_file(argv[1]); if (!file) { std::cerr << "Failed to open file " << argv[1] << ": " << pod5_get_error_string() << "\n"; return EXIT_FAILURE; @@ -41,7 +42,7 @@ int main(int argc, char** argv) { search_uuids.push_back(boost::lexical_cast(line)); } std::cout << " Read " << search_uuids.size() << " ids from the text file\n"; - } catch (std::exception const& e) { + } catch (std::exception const & e) { std::cerr << "Failed to parse UUID values from " << input_path << ": " << e.what() << "\n"; } @@ -53,9 +54,15 @@ int main(int argc, char** argv) { std::vector traversal_batch_counts(batch_count); std::vector traversal_row_indices(search_uuids.size()); std::size_t find_success_count = 0; - if (pod5_plan_traversal(file, (uint8_t*)search_uuids.data(), search_uuids.size(), - traversal_batch_counts.data(), traversal_row_indices.data(), - &find_success_count) != POD5_OK) { + if (pod5_plan_traversal( + file, + (uint8_t *)search_uuids.data(), + search_uuids.size(), + traversal_batch_counts.data(), + traversal_row_indices.data(), + &find_success_count) + != POD5_OK) + { std::cerr << "Failed to plan traversal of file: " << pod5_get_error_string() << "\n"; return EXIT_FAILURE; } @@ -70,7 +77,7 @@ int main(int argc, char** argv) { // Walk the suggested traversal route, storing read data. std::size_t step_index = 0; for (std::size_t batch_index = 0; batch_index < batch_count; ++batch_index) { - Pod5ReadRecordBatch_t* batch = nullptr; + Pod5ReadRecordBatch_t * batch = nullptr; if (pod5_get_read_batch(&batch, file, batch_index) != POD5_OK) { std::cerr << "Failed to get batch: " << pod5_get_error_string() << "\n"; return EXIT_FAILURE; @@ -83,8 +90,10 @@ int main(int argc, char** argv) { uint16_t read_table_version = 0; ReadBatchRowInfo_t read_data; - if (pod5_get_read_batch_row_info_data(batch, batch_row, READ_BATCH_ROW_INFO_VERSION, - &read_data, &read_table_version) != POD5_OK) { + if (pod5_get_read_batch_row_info_data( + batch, batch_row, READ_BATCH_ROW_INFO_VERSION, &read_data, &read_table_version) + != POD5_OK) + { std::cerr << "Failed to get read " << batch_row << "\n"; return EXIT_FAILURE; } diff --git a/c++/examples/find_specific_read_ids_with_signal.cpp b/c++/examples/find_specific_read_ids_with_signal.cpp index 64ecda7..fd4a62e 100644 --- a/c++/examples/find_specific_read_ids_with_signal.cpp +++ b/c++/examples/find_specific_read_ids_with_signal.cpp @@ -8,7 +8,8 @@ #include #include -int main(int argc, char** argv) { +int main(int argc, char ** argv) +{ if (argc != 3) { std::cerr << "Expected two arguments:\n" << " - an pod5 file to search\n" @@ -19,7 +20,7 @@ int main(int argc, char** argv) { pod5_init(); // Open the file ready for walking: - Pod5FileReader_t* file = pod5_open_file(argv[1]); + Pod5FileReader_t * file = pod5_open_file(argv[1]); if (!file) { std::cerr << "Failed to open file " << argv[1] << ": " << pod5_get_error_string() << "\n"; return EXIT_FAILURE; @@ -41,7 +42,7 @@ int main(int argc, char** argv) { search_uuids.push_back(boost::lexical_cast(line)); } std::cout << " Read " << search_uuids.size() << " ids from the text file\n"; - } catch (std::exception const& e) { + } catch (std::exception const & e) { std::cerr << "Failed to parse UUID values from " << input_path << ": " << e.what() << "\n"; } @@ -53,9 +54,15 @@ int main(int argc, char** argv) { std::vector traversal_batch_counts(batch_count); std::vector traversal_row_indices(search_uuids.size()); std::size_t find_success_count = 0; - if (pod5_plan_traversal(file, (uint8_t*)search_uuids.data(), search_uuids.size(), - traversal_batch_counts.data(), traversal_row_indices.data(), - &find_success_count) != POD5_OK) { + if (pod5_plan_traversal( + file, + (uint8_t *)search_uuids.data(), + search_uuids.size(), + traversal_batch_counts.data(), + traversal_row_indices.data(), + &find_success_count) + != POD5_OK) + { std::cerr << "Failed to plan traversal of file: " << pod5_get_error_string() << "\n"; return EXIT_FAILURE; } @@ -72,7 +79,7 @@ int main(int argc, char** argv) { // Walk the suggested traversal route, storing read data. std::size_t step_index = 0; for (std::size_t batch_index = 0; batch_index < batch_count; ++batch_index) { - Pod5ReadRecordBatch_t* batch = nullptr; + Pod5ReadRecordBatch_t * batch = nullptr; if (pod5_get_read_batch(&batch, file, batch_index) != POD5_OK) { std::cerr << "Failed to get batch: " << pod5_get_error_string() << "\n"; return EXIT_FAILURE; @@ -85,8 +92,10 @@ int main(int argc, char** argv) { uint16_t read_table_version = 0; ReadBatchRowInfo_t read_data; - if (pod5_get_read_batch_row_info_data(batch, batch_row, READ_BATCH_ROW_INFO_VERSION, - &read_data, &read_table_version) != POD5_OK) { + if (pod5_get_read_batch_row_info_data( + batch, batch_row, READ_BATCH_ROW_INFO_VERSION, &read_data, &read_table_version) + != POD5_OK) + { std::cerr << "Failed to get read " << batch_row << "\n"; return EXIT_FAILURE; } diff --git a/c++/pod5_format/async_signal_loader.cpp b/c++/pod5_format/async_signal_loader.cpp index 37d4de4..c11971a 100644 --- a/c++/pod5_format/async_signal_loader.cpp +++ b/c++/pod5_format/async_signal_loader.cpp @@ -4,26 +4,28 @@ namespace pod5 { const std::size_t AsyncSignalLoader::MINIMUM_JOB_SIZE = 50; -AsyncSignalLoader::AsyncSignalLoader(std::shared_ptr const& reader, - SamplesMode samples_mode, - gsl::span const& batch_counts, - gsl::span const& batch_rows, - std::size_t worker_count, - std::size_t max_pending_batches) - : m_reader(reader), - m_samples_mode(samples_mode), - m_max_pending_batches(max_pending_batches), - m_reads_batch_count(m_reader->num_read_record_batches()), - m_batch_counts(batch_counts), - m_total_batch_count_so_far(0), - m_batch_rows(batch_rows), - m_worker_job_size(std::max( - MINIMUM_JOB_SIZE, - m_batch_rows.size() / (m_reads_batch_count * worker_count * 2))), - m_current_batch(0), - m_finished(false), - m_has_error(false), - m_batches_size(0) { +AsyncSignalLoader::AsyncSignalLoader( + std::shared_ptr const & reader, + SamplesMode samples_mode, + gsl::span const & batch_counts, + gsl::span const & batch_rows, + std::size_t worker_count, + std::size_t max_pending_batches) +: m_reader(reader) +, m_samples_mode(samples_mode) +, m_max_pending_batches(max_pending_batches) +, m_reads_batch_count(m_reader->num_read_record_batches()) +, m_batch_counts(batch_counts) +, m_total_batch_count_so_far(0) +, m_batch_rows(batch_rows) +, m_worker_job_size(std::max( + MINIMUM_JOB_SIZE, + m_batch_rows.size() / (m_reads_batch_count * worker_count * 2))) +, m_current_batch(0) +, m_finished(false) +, m_has_error(false) +, m_batches_size(0) +{ // Setup first batch: { std::unique_lock l(m_worker_sync); @@ -39,7 +41,8 @@ AsyncSignalLoader::AsyncSignalLoader(std::shared_ptr const& re } } -AsyncSignalLoader::~AsyncSignalLoader() { +AsyncSignalLoader::~AsyncSignalLoader() +{ m_finished = true; // Wait for all workers to complete: for (std::size_t i = 0; i < m_workers.size(); ++i) { @@ -48,7 +51,8 @@ AsyncSignalLoader::~AsyncSignalLoader() { } Result> AsyncSignalLoader::release_next_batch( - boost::optional timeout) { + boost::optional timeout) +{ std::shared_ptr batch; // Return any error, if one has occurred: @@ -61,8 +65,9 @@ Result> AsyncSignalLoader::release_next_b std::unique_lock l(m_batches_sync); // Wait until there is a batch available: m_batch_done.wait_until( - l, timeout.get_value_or(std::chrono::steady_clock::now() + std::chrono::seconds(5)), - [&] { return m_batches.size() || m_finished || m_has_error; }); + l, + timeout.get_value_or(std::chrono::steady_clock::now() + std::chrono::seconds(5)), + [&] { return m_batches.size() || m_finished || m_has_error; }); // Grab a batch if one exists (note error or user destroying us might have happened instead): if (!m_batches.empty()) { @@ -97,13 +102,15 @@ Result> AsyncSignalLoader::release_next_b return nullptr; } -void AsyncSignalLoader::set_error(pod5::Status status) { +void AsyncSignalLoader::set_error(pod5::Status status) +{ assert(!status.ok()); m_error = status; m_has_error = true; } -void AsyncSignalLoader::run_worker() { +void AsyncSignalLoader::run_worker() +{ // Continue to work while there is work to do, and no error has occurred while (!m_finished && !m_has_error) { std::shared_ptr batch; @@ -157,7 +164,7 @@ void AsyncSignalLoader::run_worker() { // Now execute the work, for all the rows we said we would: std::uint32_t const row_end = - std::min(row_start + m_worker_job_size, batch->job_row_count()); + std::min(row_start + m_worker_job_size, batch->job_row_count()); do_work(batch, row_start, row_end); @@ -166,9 +173,11 @@ void AsyncSignalLoader::run_worker() { } } -void AsyncSignalLoader::do_work(std::shared_ptr const& batch, - std::uint32_t row_start, - std::uint32_t row_end) { +void AsyncSignalLoader::do_work( + std::shared_ptr const & batch, + std::uint32_t row_start, + std::uint32_t row_end) +{ // First secure the sample counts column for the batch we are processing: auto signal_column = batch->read_batch().signal_column(); @@ -178,9 +187,9 @@ void AsyncSignalLoader::do_work(std::shared_ptr const& b auto const actual_batch_row = batch->get_batch_row_to_query(i); // Get the signal row data for the read: auto const signal_rows = std::static_pointer_cast( - signal_column->value_slice(actual_batch_row)); + signal_column->value_slice(actual_batch_row)); auto const signal_rows_span = - gsl::make_span(signal_rows->raw_values(), signal_rows->length()); + gsl::make_span(signal_rows->raw_values(), signal_rows->length()); // Find the sample count for these rows: auto sample_count_result = m_reader->extract_sample_count(signal_rows_span); @@ -196,7 +205,7 @@ void AsyncSignalLoader::do_work(std::shared_ptr const& b if (m_samples_mode == SamplesMode::Samples) { samples.resize(sample_count); auto samples_result = - m_reader->extract_samples(signal_rows_span, gsl::make_span(samples)); + m_reader->extract_samples(signal_rows_span, gsl::make_span(samples)); if (!samples_result.ok()) { m_error = samples_result; m_has_error = true; @@ -210,7 +219,8 @@ void AsyncSignalLoader::do_work(std::shared_ptr const& b } } -Status AsyncSignalLoader::setup_next_in_progress_batch(std::unique_lock& lock) { +Status AsyncSignalLoader::setup_next_in_progress_batch(std::unique_lock & lock) +{ assert(!m_in_progress_batch); ARROW_ASSIGN_OR_RAISE(auto read_batch, m_reader->read_read_record_batch(m_current_batch)); std::size_t row_count = read_batch.num_rows(); @@ -224,11 +234,12 @@ Status AsyncSignalLoader::setup_next_in_progress_batch(std::unique_lock( - m_current_batch, row_count, next_specific_batch_rows, std::move(read_batch)); + m_current_batch, row_count, next_specific_batch_rows, std::move(read_batch)); return Status::OK(); } -void AsyncSignalLoader::release_in_progress_batch() { +void AsyncSignalLoader::release_in_progress_batch() +{ if (m_in_progress_batch) { assert(!m_in_progress_batch->has_work_left()); std::lock_guard l(m_batches_sync); diff --git a/c++/pod5_format/async_signal_loader.h b/c++/pod5_format/async_signal_loader.h index 7492bf4..20f744b 100644 --- a/c++/pod5_format/async_signal_loader.h +++ b/c++/pod5_format/async_signal_loader.h @@ -20,19 +20,23 @@ namespace pod5 { class POD5_FORMAT_EXPORT CachedBatchSignalData { public: CachedBatchSignalData(std::uint32_t batch_index, std::size_t entry_count) - : m_batch_index(batch_index), m_sample_counts(entry_count), m_samples(entry_count) {} + : m_batch_index(batch_index) + , m_sample_counts(entry_count) + , m_samples(entry_count) + { + } std::uint32_t batch_index() const { return m_batch_index; } /// Find a list of sample counts for all requested batch rows. - std::vector const& sample_count() const { return m_sample_counts; } + std::vector const & sample_count() const { return m_sample_counts; } /// Find a list of signal samples counts for all requested batch rows. - std::vector> const& samples() const { return m_samples; } + std::vector> const & samples() const { return m_samples; } - void set_samples(std::size_t row, - std::uint64_t sample_count, - std::vector&& samples) { + void + set_samples(std::size_t row, std::uint64_t sample_count, std::vector && samples) + { m_sample_counts[row] = sample_count; m_samples[row] = std::move(samples); } @@ -45,31 +49,35 @@ class POD5_FORMAT_EXPORT CachedBatchSignalData { class POD5_FORMAT_EXPORT SignalCacheWorkPackage { public: - SignalCacheWorkPackage(std::uint32_t batch_index, - std::size_t job_row_count, - gsl::span const& specific_job_rows, - pod5::ReadTableRecordBatch&& read_batch) - : m_job_row_count(job_row_count), - m_specific_job_rows(specific_job_rows), - m_next_row_to_start(0), - m_completed_rows(0), - m_cached_data(std::make_unique(batch_index, m_job_row_count)), - m_read_batch(std::move(read_batch)) {} + SignalCacheWorkPackage( + std::uint32_t batch_index, + std::size_t job_row_count, + gsl::span const & specific_job_rows, + pod5::ReadTableRecordBatch && read_batch) + : m_job_row_count(job_row_count) + , m_specific_job_rows(specific_job_rows) + , m_next_row_to_start(0) + , m_completed_rows(0) + , m_cached_data(std::make_unique(batch_index, m_job_row_count)) + , m_read_batch(std::move(read_batch)) + { + } std::uint32_t job_row_count() const { return m_job_row_count; } - void set_samples(std::size_t row, - std::uint64_t sample_count, - std::vector&& samples) { + void + set_samples(std::size_t row, std::uint64_t sample_count, std::vector && samples) + { m_cached_data->set_samples(row, sample_count, std::move(samples)); } std::unique_ptr release_data() { return std::move(m_cached_data); } - pod5::ReadTableRecordBatch const& read_batch() const { return m_read_batch; } + pod5::ReadTableRecordBatch const & read_batch() const { return m_read_batch; } // Find the actual batch row to query, for a given job row index. - std::uint32_t get_batch_row_to_query(std::uint32_t job_row_index) const { + std::uint32_t get_batch_row_to_query(std::uint32_t job_row_index) const + { // We allow the caller to specify a subset of batch rows to iterate: if (!m_specific_job_rows.empty()) { return m_specific_job_rows[job_row_index]; @@ -78,14 +86,17 @@ class POD5_FORMAT_EXPORT SignalCacheWorkPackage { return job_row_index; } - std::uint32_t start_rows(std::unique_lock& l, std::size_t row_count) { + std::uint32_t start_rows(std::unique_lock & l, std::size_t row_count) + { auto row = m_next_row_to_start; m_next_row_to_start += row_count; return row; } void complete_rows(std::uint32_t row_count) { m_completed_rows += row_count; } + bool has_work_left() const { return m_next_row_to_start < m_job_row_count; } + bool is_complete() const { return m_completed_rows.load() >= m_job_row_count; } private: @@ -108,12 +119,13 @@ class POD5_FORMAT_EXPORT AsyncSignalLoader { Samples, }; - AsyncSignalLoader(std::shared_ptr const& reader, - SamplesMode samples_mode, - gsl::span const& batch_counts, - gsl::span const& batch_rows, - std::size_t worker_count = std::thread::hardware_concurrency(), - std::size_t max_pending_batches = 10); + AsyncSignalLoader( + std::shared_ptr const & reader, + SamplesMode samples_mode, + gsl::span const & batch_counts, + gsl::span const & batch_rows, + std::size_t worker_count = std::thread::hardware_concurrency(), + std::size_t max_pending_batches = 10); ~AsyncSignalLoader(); @@ -123,22 +135,23 @@ class POD5_FORMAT_EXPORT AsyncSignalLoader { /// Get the next batch of loaded signal, always returns the consecutive next signal batch /// \note Returns nullptr when timeoout occurs, or if all data is exhausted. Result> release_next_batch( - boost::optional timeout = boost::none); + boost::optional timeout = boost::none); private: /// Set an error code that will stop all async loading and return an error to the caller. void set_error(pod5::Status status); void run_worker(); - void do_work(std::shared_ptr const& batch, - std::uint32_t row_start, - std::uint32_t row_end); + void do_work( + std::shared_ptr const & batch, + std::uint32_t row_start, + std::uint32_t row_end); /// Setup a new batch for in progress work to contain. /// \param lock A lock held on m_worker_sync. /// \note There must not be a batch already in progress. /// \note m_current_batch is used as the index of the next batch to begin. - Status setup_next_in_progress_batch(std::unique_lock& lock); + Status setup_next_in_progress_batch(std::unique_lock & lock); /// Release the currently in progress batch to readers, if it exists. /// \note This call locks m_batches_sync internally. diff --git a/c++/pod5_format/c_api.cpp b/c++/pod5_format/c_api.cpp index cbc0766..da14efd 100644 --- a/c++/pod5_format/c_api.cpp +++ b/c++/pod5_format/c_api.cpp @@ -19,19 +19,25 @@ //--------------------------------------------------------------------------------------------------------------------- struct Pod5FileReader { - Pod5FileReader(std::shared_ptr&& reader_) : reader(std::move(reader_)) {} + Pod5FileReader(std::shared_ptr && reader_) : reader(std::move(reader_)) {} + std::shared_ptr reader; }; struct Pod5FileWriter { - Pod5FileWriter(std::unique_ptr&& writer_) : writer(std::move(writer_)) {} + Pod5FileWriter(std::unique_ptr && writer_) : writer(std::move(writer_)) {} + std::unique_ptr writer; }; struct Pod5ReadRecordBatch { - Pod5ReadRecordBatch(pod5::ReadTableRecordBatch&& batch_, - std::shared_ptr const& reader) - : batch(std::move(batch_)), reader(reader) {} + Pod5ReadRecordBatch( + pod5::ReadTableRecordBatch && batch_, + std::shared_ptr const & reader) + : batch(std::move(batch_)) + , reader(reader) + { + } pod5::ReadTableRecordBatch batch; std::shared_ptr reader; @@ -43,14 +49,16 @@ pod5_error_t g_pod5_error_no; std::string g_pod5_error_string; } // namespace -extern "C" void pod5_set_error(arrow::Status status) { +extern "C" void pod5_set_error(arrow::Status status) +{ g_pod5_error_no = (pod5_error_t)status.code(); g_pod5_error_string = status.ToString(); } namespace { -void pod5_reset_error() { +void pod5_reset_error() +{ g_pod5_error_no = pod5_error_t::POD5_OK; g_pod5_error_string.clear(); } @@ -65,19 +73,20 @@ void pod5_reset_error() { } while (0) #define POD5_C_ASSIGN_OR_RAISE_IMPL(result_name, lhs, rexpr) \ - auto&& result_name = (rexpr); \ + auto && result_name = (rexpr); \ if (!(result_name).ok()) { \ pod5_set_error((result_name).status()); \ return g_pod5_error_no; \ } \ lhs = std::move(result_name).ValueUnsafe(); -#define POD5_C_ASSIGN_OR_RAISE(lhs, rexpr) \ - POD5_C_ASSIGN_OR_RAISE_IMPL(ARROW_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, \ - rexpr); +#define POD5_C_ASSIGN_OR_RAISE(lhs, rexpr) \ + POD5_C_ASSIGN_OR_RAISE_IMPL( \ + ARROW_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, rexpr); //--------------------------------------------------------------------------------------------------------------------- -bool check_string_not_empty(char const* str) { +bool check_string_not_empty(char const * str) +{ if (!str) { pod5_set_error(arrow::Status::Invalid("null string passed to C API")); return false; @@ -91,7 +100,8 @@ bool check_string_not_empty(char const* str) { return true; } -bool check_not_null(void const* ptr) { +bool check_not_null(void const * ptr) +{ if (!ptr) { pod5_set_error(arrow::Status::Invalid("null passed to C API")); return false; @@ -99,7 +109,8 @@ bool check_not_null(void const* ptr) { return true; } -bool check_file_not_null(void const* file) { +bool check_file_not_null(void const * file) +{ if (!file) { pod5_set_error(arrow::Status::Invalid("null file passed to C API")); return false; @@ -107,7 +118,8 @@ bool check_file_not_null(void const* file) { return true; } -bool check_output_pointer_not_null(void const* output) { +bool check_output_pointer_not_null(void const * output) +{ if (!output) { pod5_set_error(arrow::Status::Invalid("null output parameter passed to C API")); return false; @@ -116,7 +128,8 @@ bool check_output_pointer_not_null(void const* output) { } //--------------------------------------------------------------------------------------------------------------------- -pod5::FileWriterOptions make_internal_writer_options(Pod5WriterOptions const* options) { +pod5::FileWriterOptions make_internal_writer_options(Pod5WriterOptions const * options) +{ pod5::FileWriterOptions internal_options; if (options) { if (options->max_signal_chunk_size != 0) { @@ -142,13 +155,15 @@ pod5::FileWriterOptions make_internal_writer_options(Pod5WriterOptions const* op extern "C" { //--------------------------------------------------------------------------------------------------------------------- -pod5_error_t pod5_init() { +pod5_error_t pod5_init() +{ pod5_reset_error(); POD5_C_RETURN_NOT_OK(pod5::register_extension_types()); return POD5_OK; } -pod5_error_t pod5_terminate() { +pod5_error_t pod5_terminate() +{ pod5_reset_error(); POD5_C_RETURN_NOT_OK(pod5::unregister_extension_types()); return POD5_OK; @@ -156,10 +171,11 @@ pod5_error_t pod5_terminate() { pod5_error_t pod5_get_error_no() { return g_pod5_error_no; } -char const* pod5_get_error_string() { return g_pod5_error_string.c_str(); } +char const * pod5_get_error_string() { return g_pod5_error_string.c_str(); } //--------------------------------------------------------------------------------------------------------------------- -Pod5FileReader* pod5_open_file(char const* filename) { +Pod5FileReader * pod5_open_file(char const * filename) +{ pod5_reset_error(); if (!check_string_not_empty(filename)) { @@ -176,7 +192,8 @@ Pod5FileReader* pod5_open_file(char const* filename) { return reader.release(); } -pod5_error_t pod5_close_and_free_reader(Pod5FileReader* file) { +pod5_error_t pod5_close_and_free_reader(Pod5FileReader * file) +{ pod5_reset_error(); std::unique_ptr ptr{file}; @@ -184,7 +201,8 @@ pod5_error_t pod5_close_and_free_reader(Pod5FileReader* file) { return POD5_OK; } -pod5_error_t pod5_get_file_info(Pod5FileReader_t* reader, FileInfo* file_info) { +pod5_error_t pod5_get_file_info(Pod5FileReader_t * reader, FileInfo * file_info) +{ pod5_reset_error(); if (!check_file_not_null(reader) || !check_output_pointer_not_null(file_info)) { @@ -193,8 +211,10 @@ pod5_error_t pod5_get_file_info(Pod5FileReader_t* reader, FileInfo* file_info) { auto const metadata = reader->reader->schema_metadata(); - std::copy(metadata.file_identifier.begin(), metadata.file_identifier.end(), - file_info->file_identifier); + std::copy( + metadata.file_identifier.begin(), + metadata.file_identifier.end(), + file_info->file_identifier); file_info->version.major = metadata.writing_pod5_version.major_version(); file_info->version.minor = metadata.writing_pod5_version.minor_version(); @@ -203,14 +223,16 @@ pod5_error_t pod5_get_file_info(Pod5FileReader_t* reader, FileInfo* file_info) { return POD5_OK; } -pod5_error_t pod5_get_file_read_table_location(Pod5FileReader_t* reader, - EmbeddedFileData_t* file_data) { +pod5_error_t pod5_get_file_read_table_location( + Pod5FileReader_t * reader, + EmbeddedFileData_t * file_data) +{ pod5_reset_error(); if (!check_file_not_null(reader) || !check_output_pointer_not_null(file_data)) { return g_pod5_error_no; } - auto const& read_table_location = reader->reader->read_table_location(); + auto const & read_table_location = reader->reader->read_table_location(); file_data->offset = read_table_location.offset; file_data->offset = read_table_location.offset; @@ -218,8 +240,10 @@ pod5_error_t pod5_get_file_read_table_location(Pod5FileReader_t* reader, return POD5_OK; } -pod5_error_t pod5_get_file_signal_table_location(Pod5FileReader_t* reader, - EmbeddedFileData_t* file_data) { +pod5_error_t pod5_get_file_signal_table_location( + Pod5FileReader_t * reader, + EmbeddedFileData_t * file_data) +{ pod5_reset_error(); if (!check_file_not_null(reader) || !check_output_pointer_not_null(file_data)) { @@ -232,8 +256,10 @@ pod5_error_t pod5_get_file_signal_table_location(Pod5FileReader_t* reader, return POD5_OK; } -pod5_error_t pod5_get_file_run_info_table_location(Pod5FileReader_t* reader, - EmbeddedFileData_t* file_data) { +pod5_error_t pod5_get_file_run_info_table_location( + Pod5FileReader_t * reader, + EmbeddedFileData_t * file_data) +{ pod5_reset_error(); if (!check_file_not_null(reader) || !check_output_pointer_not_null(file_data)) { @@ -246,29 +272,32 @@ pod5_error_t pod5_get_file_run_info_table_location(Pod5FileReader_t* reader, return POD5_OK; } -pod5_error_t pod5_plan_traversal(Pod5FileReader_t* reader, - uint8_t const* read_id_array, - size_t read_id_count, - uint32_t* batch_counts, - uint32_t* batch_rows, - size_t* find_success_count_out) { +pod5_error_t pod5_plan_traversal( + Pod5FileReader_t * reader, + uint8_t const * read_id_array, + size_t read_id_count, + uint32_t * batch_counts, + uint32_t * batch_rows, + size_t * find_success_count_out) +{ pod5_reset_error(); - if (!check_file_not_null(reader) || !check_not_null(read_id_array) || - !check_output_pointer_not_null(batch_counts) || - !check_output_pointer_not_null(batch_rows)) { + if (!check_file_not_null(reader) || !check_not_null(read_id_array) + || !check_output_pointer_not_null(batch_counts) + || !check_output_pointer_not_null(batch_rows)) + { return g_pod5_error_no; } - auto search_input = pod5::ReadIdSearchInput(gsl::make_span( - reinterpret_cast(read_id_array), read_id_count)); + auto search_input = pod5::ReadIdSearchInput( + gsl::make_span(reinterpret_cast(read_id_array), read_id_count)); POD5_C_ASSIGN_OR_RAISE( - auto find_success_count, - reader->reader->search_for_read_ids( - search_input, - gsl::make_span(batch_counts, reader->reader->num_read_record_batches()), - gsl::make_span(batch_rows, read_id_count))); + auto find_success_count, + reader->reader->search_for_read_ids( + search_input, + gsl::make_span(batch_counts, reader->reader->num_read_record_batches()), + gsl::make_span(batch_rows, read_id_count))); if (find_success_count_out) { *find_success_count_out = find_success_count; @@ -277,7 +306,8 @@ pod5_error_t pod5_plan_traversal(Pod5FileReader_t* reader, return POD5_OK; } -pod5_error_t pod5_get_read_batch_count(size_t* count, Pod5FileReader* reader) { +pod5_error_t pod5_get_read_batch_count(size_t * count, Pod5FileReader * reader) +{ pod5_reset_error(); if (!check_file_not_null(reader) || !check_output_pointer_not_null(count)) { @@ -288,9 +318,9 @@ pod5_error_t pod5_get_read_batch_count(size_t* count, Pod5FileReader* reader) { return POD5_OK; } -pod5_error_t pod5_get_read_batch(Pod5ReadRecordBatch** batch, - Pod5FileReader* reader, - size_t index) { +pod5_error_t +pod5_get_read_batch(Pod5ReadRecordBatch ** batch, Pod5FileReader * reader, size_t index) +{ pod5_reset_error(); if (!check_file_not_null(reader) || !check_output_pointer_not_null(batch)) { @@ -300,13 +330,14 @@ pod5_error_t pod5_get_read_batch(Pod5ReadRecordBatch** batch, POD5_C_ASSIGN_OR_RAISE(auto internal_batch, reader->reader->read_read_record_batch(index)); auto wrapped_batch = - std::make_unique(std::move(internal_batch), reader->reader); + std::make_unique(std::move(internal_batch), reader->reader); *batch = wrapped_batch.release(); return POD5_OK; } -pod5_error_t pod5_free_read_batch(Pod5ReadRecordBatch* batch) { +pod5_error_t pod5_free_read_batch(Pod5ReadRecordBatch * batch) +{ pod5_reset_error(); if (!check_not_null(batch)) { @@ -318,7 +349,8 @@ pod5_error_t pod5_free_read_batch(Pod5ReadRecordBatch* batch) { return POD5_OK; } -pod5_error_t pod5_get_read_batch_row_count(size_t* count, Pod5ReadRecordBatch* batch) { +pod5_error_t pod5_get_read_batch_row_count(size_t * count, Pod5ReadRecordBatch * batch) +{ pod5_reset_error(); if (!check_not_null(batch) || !check_output_pointer_not_null(count)) { @@ -329,20 +361,23 @@ pod5_error_t pod5_get_read_batch_row_count(size_t* count, Pod5ReadRecordBatch* b return POD5_OK; } -pod5_error_t pod5_get_read_batch_row_info_data(Pod5ReadRecordBatch_t* batch, - size_t row, - uint16_t struct_version, - void* row_data, - uint16_t* read_table_version) { +pod5_error_t pod5_get_read_batch_row_info_data( + Pod5ReadRecordBatch_t * batch, + size_t row, + uint16_t struct_version, + void * row_data, + uint16_t * read_table_version) +{ if (!check_not_null(batch) || !check_output_pointer_not_null(row_data)) { return g_pod5_error_no; } - static_assert(READ_BATCH_ROW_INFO_VERSION == READ_BATCH_ROW_INFO_VERSION_3, - "New versions must be explicitly loaded"); + static_assert( + READ_BATCH_ROW_INFO_VERSION == READ_BATCH_ROW_INFO_VERSION_3, + "New versions must be explicitly loaded"); if (struct_version == READ_BATCH_ROW_INFO_VERSION_3) { - auto typed_row_data = static_cast(row_data); + auto typed_row_data = static_cast(row_data); POD5_C_ASSIGN_OR_RAISE(auto cols, batch->batch.columns()); @@ -362,7 +397,7 @@ pod5_error_t pod5_get_read_batch_row_info_data(Pod5ReadRecordBatch_t* batch, typed_row_data->calibration_offset = cols.calibration_offset->Value(row); typed_row_data->calibration_scale = cols.calibration_scale->Value(row); auto end_reason_col = - std::static_pointer_cast(cols.end_reason->indices()); + std::static_pointer_cast(cols.end_reason->indices()); typed_row_data->end_reason = end_reason_col->Value(row); typed_row_data->end_reason_forced = cols.end_reason_forced->Value(row); auto run_info_col = std::static_pointer_cast(cols.run_info->indices()); @@ -379,17 +414,19 @@ pod5_error_t pod5_get_read_batch_row_info_data(Pod5ReadRecordBatch_t* batch, typed_row_data->num_samples = cols.num_samples->Value(row); } else { pod5_set_error( - arrow::Status::Invalid("Invalid struct version '", struct_version, "' passed")); + arrow::Status::Invalid("Invalid struct version '", struct_version, "' passed")); return g_pod5_error_no; } return POD5_OK; } -pod5_error_t pod5_get_signal_row_indices(Pod5ReadRecordBatch* batch, - size_t row, - int64_t signal_row_indices_count, - uint64_t* signal_row_indices) { +pod5_error_t pod5_get_signal_row_indices( + Pod5ReadRecordBatch * batch, + size_t row, + int64_t signal_row_indices_count, + uint64_t * signal_row_indices) +{ pod5_reset_error(); if (!check_not_null(batch) || !check_output_pointer_not_null(signal_row_indices)) { @@ -397,13 +434,15 @@ pod5_error_t pod5_get_signal_row_indices(Pod5ReadRecordBatch* batch, } auto const signal_col = batch->batch.signal_column(); - auto const& row_data = - std::static_pointer_cast(signal_col->value_slice(row)); + auto const & row_data = + std::static_pointer_cast(signal_col->value_slice(row)); if (signal_row_indices_count != row_data->length()) { - pod5_set_error(pod5::Status::Invalid("Incorrect number of signal indices, expected ", - row_data->length(), " received ", - signal_row_indices_count)); + pod5_set_error(pod5::Status::Invalid( + "Incorrect number of signal indices, expected ", + row_data->length(), + " received ", + signal_row_indices_count)); return g_pod5_error_no; } @@ -414,9 +453,11 @@ pod5_error_t pod5_get_signal_row_indices(Pod5ReadRecordBatch* batch, return POD5_OK; } -pod5_error_t pod5_get_calibration_extra_info(Pod5ReadRecordBatch_t* batch, - size_t row, - CalibrationExtraData_t* calibration_extra_data) { +pod5_error_t pod5_get_calibration_extra_info( + Pod5ReadRecordBatch_t * batch, + size_t row, + CalibrationExtraData_t * calibration_extra_data) +{ pod5_reset_error(); if (!check_not_null(batch) || !check_output_pointer_not_null(calibration_extra_data)) { @@ -427,10 +468,10 @@ pod5_error_t pod5_get_calibration_extra_info(Pod5ReadRecordBatch_t* batch, auto scale = cols.calibration_scale->Value(row); auto const run_info_dict_index = - std::static_pointer_cast(cols.run_info->indices())->Value(row); + std::static_pointer_cast(cols.run_info->indices())->Value(row); - POD5_C_ASSIGN_OR_RAISE(auto const acquisition_id, - batch->batch.get_run_info(run_info_dict_index)); + POD5_C_ASSIGN_OR_RAISE( + auto const acquisition_id, batch->batch.get_run_info(run_info_dict_index)); POD5_C_ASSIGN_OR_RAISE(auto run_info_data, batch->reader->find_run_info(acquisition_id)); calibration_extra_data->digitisation = run_info_data->adc_max - run_info_data->adc_min + 1; @@ -441,12 +482,13 @@ pod5_error_t pod5_get_calibration_extra_info(Pod5ReadRecordBatch_t* batch, struct RunInfoDataCHelper : public RunInfoDictData { struct InternalMapHelper { - std::vector keys; - std::vector values; + std::vector keys; + std::vector values; }; - RunInfoDataCHelper(std::shared_ptr const& internal_data_) - : internal_data(internal_data_) { + RunInfoDataCHelper(std::shared_ptr const & internal_data_) + : internal_data(internal_data_) + { acquisition_id = internal_data->acquisition_id.c_str(); acquisition_start_time_ms = internal_data->acquisition_start_time; adc_max = internal_data->adc_max; @@ -469,10 +511,11 @@ struct RunInfoDataCHelper : public RunInfoDictData { tracking_id = map_to_c(internal_data->tracking_id, tracking_id_helper); } - KeyValueData map_to_c(pod5::RunInfoData::MapType const& map, InternalMapHelper& helper) { + KeyValueData map_to_c(pod5::RunInfoData::MapType const & map, InternalMapHelper & helper) + { helper.keys.reserve(map.size()); helper.values.reserve(map.size()); - for (auto const& item : map) { + for (auto const & item : map) { helper.keys.push_back(item.first.c_str()); helper.values.push_back(item.second.c_str()); } @@ -489,9 +532,9 @@ struct RunInfoDataCHelper : public RunInfoDictData { InternalMapHelper tracking_id_helper; }; -pod5_error_t pod5_get_run_info(Pod5ReadRecordBatch* batch, - int16_t run_info, - RunInfoDictData** run_info_data) { +pod5_error_t +pod5_get_run_info(Pod5ReadRecordBatch * batch, int16_t run_info, RunInfoDictData ** run_info_data) +{ pod5_reset_error(); if (!check_not_null(batch) || !check_output_pointer_not_null(run_info_data)) { @@ -506,29 +549,33 @@ pod5_error_t pod5_get_run_info(Pod5ReadRecordBatch* batch, return POD5_OK; } -POD5_FORMAT_EXPORT pod5_error_t pod5_release_run_info(RunInfoDictData* run_info_data) { +POD5_FORMAT_EXPORT pod5_error_t pod5_release_run_info(RunInfoDictData * run_info_data) +{ pod5_reset_error(); if (!check_not_null(run_info_data)) { return g_pod5_error_no; } - std::unique_ptr helper(static_cast(run_info_data)); + std::unique_ptr helper(static_cast(run_info_data)); helper.reset(); return POD5_OK; } -pod5_error_t pod5_get_end_reason(Pod5ReadRecordBatch_t* batch, - int16_t end_reason, - pod5_end_reason* end_reason_value, - char* end_reason_string_value, - size_t* end_reason_string_value_size) { +pod5_error_t pod5_get_end_reason( + Pod5ReadRecordBatch_t * batch, + int16_t end_reason, + pod5_end_reason * end_reason_value, + char * end_reason_string_value, + size_t * end_reason_string_value_size) +{ pod5_reset_error(); - if (!check_output_pointer_not_null(end_reason_value) || - !check_output_pointer_not_null(end_reason_string_value) || - !check_output_pointer_not_null(end_reason_string_value_size)) { + if (!check_output_pointer_not_null(end_reason_value) + || !check_output_pointer_not_null(end_reason_string_value) + || !check_output_pointer_not_null(end_reason_string_value_size)) + { return g_pod5_error_no; } @@ -565,14 +612,17 @@ pod5_error_t pod5_get_end_reason(Pod5ReadRecordBatch_t* batch, return POD5_OK; } -pod5_error_t pod5_get_pore_type(Pod5ReadRecordBatch_t* batch, - int16_t pore_type, - char* pore_type_string_value, - size_t* pore_type_string_value_size) { +pod5_error_t pod5_get_pore_type( + Pod5ReadRecordBatch_t * batch, + int16_t pore_type, + char * pore_type_string_value, + size_t * pore_type_string_value_size) +{ pod5_reset_error(); - if (!check_output_pointer_not_null(pore_type_string_value) || - !check_output_pointer_not_null(pore_type_string_value_size)) { + if (!check_output_pointer_not_null(pore_type_string_value) + || !check_output_pointer_not_null(pore_type_string_value_size)) + { return g_pod5_error_no; } @@ -589,15 +639,17 @@ pod5_error_t pod5_get_pore_type(Pod5ReadRecordBatch_t* batch, class SignalRowInfoCHelper : public SignalRowInfo { public: - SignalRowInfoCHelper(pod5::SignalTableRecordBatch const& b) : batch(b) {} + SignalRowInfoCHelper(pod5::SignalTableRecordBatch const & b) : batch(b) {} pod5::SignalTableRecordBatch batch; }; -pod5_error_t pod5_get_signal_row_info(Pod5FileReader* reader, - size_t signal_rows_count, - uint64_t* signal_rows, - SignalRowInfo** signal_row_info) { +pod5_error_t pod5_get_signal_row_info( + Pod5FileReader * reader, + size_t signal_rows_count, + uint64_t * signal_rows, + SignalRowInfo ** signal_row_info) +{ pod5_reset_error(); if (!check_not_null(reader) || !check_output_pointer_not_null(signal_row_info)) { @@ -609,14 +661,14 @@ pod5_error_t pod5_get_signal_row_info(Pod5FileReader* reader, std::sort(signal_rows_sorted.begin(), signal_rows_sorted.end()); // Then loop all rows, forward. - for (std::size_t completed_rows = 0; - completed_rows < - signal_rows_sorted.size();) { // No increment here, we do it below when we succeed. + for (std::size_t completed_rows = 0; completed_rows < signal_rows_sorted.size();) + { // No increment here, we do it below when we succeed. auto const start_row = signal_rows_sorted[completed_rows]; std::size_t batch_row = 0; - POD5_C_ASSIGN_OR_RAISE(std::size_t row_batch, - (reader->reader->signal_batch_for_row_id(start_row, &batch_row))); + POD5_C_ASSIGN_OR_RAISE( + std::size_t row_batch, + (reader->reader->signal_batch_for_row_id(start_row, &batch_row))); POD5_C_ASSIGN_OR_RAISE(auto batch, reader->reader->read_signal_record_batch(row_batch)); auto output = std::make_unique(batch); @@ -635,81 +687,89 @@ pod5_error_t pod5_get_signal_row_info(Pod5FileReader* reader, return POD5_OK; } -pod5_error_t pod5_free_signal_row_info(size_t signal_rows_count, - SignalRowInfo_t** signal_row_info) { +pod5_error_t pod5_free_signal_row_info(size_t signal_rows_count, SignalRowInfo_t ** signal_row_info) +{ for (std::size_t i = 0; i < signal_rows_count; ++i) { std::unique_ptr helper( - static_cast(signal_row_info[i])); + static_cast(signal_row_info[i])); helper.reset(); } return POD5_OK; } -pod5_error_t pod5_get_signal(Pod5FileReader* reader, - SignalRowInfo_t* row_info, - std::size_t sample_count, - std::int16_t* sample_data) { +pod5_error_t pod5_get_signal( + Pod5FileReader * reader, + SignalRowInfo_t * row_info, + std::size_t sample_count, + std::int16_t * sample_data) +{ pod5_reset_error(); - if (!check_not_null(reader) || !check_not_null(row_info) || - !check_output_pointer_not_null(sample_data)) { + if (!check_not_null(reader) || !check_not_null(row_info) + || !check_output_pointer_not_null(sample_data)) + { return g_pod5_error_no; } - SignalRowInfoCHelper* row_info_data = static_cast(row_info); + SignalRowInfoCHelper * row_info_data = static_cast(row_info); POD5_C_RETURN_NOT_OK(row_info_data->batch.extract_signal_row( - row_info->batch_row_index, gsl::make_span(sample_data, sample_count))); + row_info->batch_row_index, gsl::make_span(sample_data, sample_count))); return POD5_OK; } -pod5_error_t pod5_get_read_complete_sample_count(Pod5FileReader_t* reader, - Pod5ReadRecordBatch_t* batch, - size_t batch_row, - size_t* sample_count) { +pod5_error_t pod5_get_read_complete_sample_count( + Pod5FileReader_t * reader, + Pod5ReadRecordBatch_t * batch, + size_t batch_row, + size_t * sample_count) +{ pod5_reset_error(); if (!check_not_null(reader) || !check_output_pointer_not_null(sample_count)) { return g_pod5_error_no; } - auto const& signal_col = batch->batch.signal_column(); - auto const& signal_rows = - std::static_pointer_cast(signal_col->value_slice(batch_row)); + auto const & signal_col = batch->batch.signal_column(); + auto const & signal_rows = + std::static_pointer_cast(signal_col->value_slice(batch_row)); std::vector output_samples; - POD5_C_ASSIGN_OR_RAISE(*sample_count, - reader->reader->extract_sample_count(gsl::make_span( - signal_rows->raw_values(), signal_rows->length()))); + POD5_C_ASSIGN_OR_RAISE( + *sample_count, + reader->reader->extract_sample_count( + gsl::make_span(signal_rows->raw_values(), signal_rows->length()))); return POD5_OK; } -pod5_error_t pod5_get_read_complete_signal(Pod5FileReader_t* reader, - Pod5ReadRecordBatch_t* batch, - size_t batch_row, - size_t sample_count, - int16_t* signal) { +pod5_error_t pod5_get_read_complete_signal( + Pod5FileReader_t * reader, + Pod5ReadRecordBatch_t * batch, + size_t batch_row, + size_t sample_count, + int16_t * signal) +{ pod5_reset_error(); if (!check_not_null(reader) || !check_output_pointer_not_null(signal)) { return g_pod5_error_no; } - auto const& signal_col = batch->batch.signal_column(); - auto const& signal_rows = - std::static_pointer_cast(signal_col->value_slice(batch_row)); + auto const & signal_col = batch->batch.signal_column(); + auto const & signal_rows = + std::static_pointer_cast(signal_col->value_slice(batch_row)); POD5_C_RETURN_NOT_OK(reader->reader->extract_samples( - gsl::make_span(signal_rows->raw_values(), signal_rows->length()), - gsl::make_span(signal, sample_count))); + gsl::make_span(signal_rows->raw_values(), signal_rows->length()), + gsl::make_span(signal, sample_count))); return POD5_OK; } //--------------------------------------------------------------------------------------------------------------------- -Pod5FileWriter* pod5_create_file(char const* filename, - char const* writer_name, - Pod5WriterOptions const* options) { +Pod5FileWriter * +pod5_create_file(char const * filename, char const * writer_name, Pod5WriterOptions const * options) +{ pod5_reset_error(); if (!check_string_not_empty(filename) || !check_string_not_empty(writer_name)) { @@ -717,7 +777,7 @@ Pod5FileWriter* pod5_create_file(char const* filename, } auto internal_writer = - pod5::create_file_writer(filename, writer_name, make_internal_writer_options(options)); + pod5::create_file_writer(filename, writer_name, make_internal_writer_options(options)); if (!internal_writer.ok()) { pod5_set_error(internal_writer.status()); return nullptr; @@ -727,7 +787,8 @@ Pod5FileWriter* pod5_create_file(char const* filename, return writer.release(); } -pod5_error_t pod5_close_and_free_writer(Pod5FileWriter* file) { +pod5_error_t pod5_close_and_free_writer(Pod5FileWriter * file) +{ pod5_reset_error(); std::unique_ptr ptr{file}; @@ -737,11 +798,13 @@ pod5_error_t pod5_close_and_free_writer(Pod5FileWriter* file) { return POD5_OK; } -pod5_error_t pod5_add_pore(int16_t* pore_index, Pod5FileWriter* file, char const* pore_type) { +pod5_error_t pod5_add_pore(int16_t * pore_index, Pod5FileWriter * file, char const * pore_type) +{ pod5_reset_error(); - if (!check_string_not_empty(pore_type) || !check_file_not_null(file) || - !check_output_pointer_not_null(pore_index)) { + if (!check_string_not_empty(pore_type) || !check_file_not_null(file) + || !check_output_pointer_not_null(pore_index)) + { return g_pod5_error_no; } @@ -749,32 +812,34 @@ pod5_error_t pod5_add_pore(int16_t* pore_index, Pod5FileWriter* file, char const return POD5_OK; } -pod5_error_t pod5_add_run_info(int16_t* run_info_index, - Pod5FileWriter* file, - char const* acquisition_id, - std::int64_t acquisition_start_time_ms, - std::int16_t adc_max, - std::int16_t adc_min, - std::size_t context_tags_count, - char const** context_tags_keys, - char const** context_tags_values, - char const* experiment_name, - char const* flow_cell_id, - char const* flow_cell_product_code, - char const* protocol_name, - char const* protocol_run_id, - std::int64_t protocol_start_time_ms, - char const* sample_id, - std::uint16_t sample_rate, - char const* sequencing_kit, - char const* sequencer_position, - char const* sequencer_position_type, - char const* software, - char const* system_name, - char const* system_type, - std::size_t tracking_id_count, - char const** tracking_id_keys, - char const** tracking_id_values) { +pod5_error_t pod5_add_run_info( + int16_t * run_info_index, + Pod5FileWriter * file, + char const * acquisition_id, + std::int64_t acquisition_start_time_ms, + std::int16_t adc_max, + std::int16_t adc_min, + std::size_t context_tags_count, + char const ** context_tags_keys, + char const ** context_tags_values, + char const * experiment_name, + char const * flow_cell_id, + char const * flow_cell_product_code, + char const * protocol_name, + char const * protocol_run_id, + std::int64_t protocol_start_time_ms, + char const * sample_id, + std::uint16_t sample_rate, + char const * sequencing_kit, + char const * sequencer_position, + char const * sequencer_position_type, + char const * software, + char const * system_name, + char const * system_type, + std::size_t tracking_id_count, + char const ** tracking_id_keys, + char const ** tracking_id_values) +{ pod5_reset_error(); if (!check_file_not_null(file)) { @@ -782,8 +847,9 @@ pod5_error_t pod5_add_run_info(int16_t* run_info_index, } auto const parse_map = - [](std::size_t tracking_id_count, char const** tracking_id_keys, - char const** tracking_id_values) -> pod5::Result { + [](std::size_t tracking_id_count, + char const ** tracking_id_keys, + char const ** tracking_id_values) -> pod5::Result { pod5::RunInfoData::MapType result; for (std::size_t i = 0; i < tracking_id_count; ++i) { auto key = tracking_id_keys[i]; @@ -797,26 +863,44 @@ pod5_error_t pod5_add_run_info(int16_t* run_info_index, return result; }; - POD5_C_ASSIGN_OR_RAISE(auto const context_tags, - parse_map(context_tags_count, context_tags_keys, context_tags_values)); - POD5_C_ASSIGN_OR_RAISE(auto const tracking_id, - parse_map(tracking_id_count, tracking_id_keys, tracking_id_values)); + POD5_C_ASSIGN_OR_RAISE( + auto const context_tags, + parse_map(context_tags_count, context_tags_keys, context_tags_values)); + POD5_C_ASSIGN_OR_RAISE( + auto const tracking_id, parse_map(tracking_id_count, tracking_id_keys, tracking_id_values)); POD5_C_ASSIGN_OR_RAISE( - *run_info_index, - file->writer->add_run_info(pod5::RunInfoData( - acquisition_id, acquisition_start_time_ms, adc_max, adc_min, context_tags, - experiment_name, flow_cell_id, flow_cell_product_code, protocol_name, - protocol_run_id, protocol_start_time_ms, sample_id, sample_rate, sequencing_kit, - sequencer_position, sequencer_position_type, software, system_name, system_type, - tracking_id))); + *run_info_index, + file->writer->add_run_info(pod5::RunInfoData( + acquisition_id, + acquisition_start_time_ms, + adc_max, + adc_min, + context_tags, + experiment_name, + flow_cell_id, + flow_cell_product_code, + protocol_name, + protocol_run_id, + protocol_start_time_ms, + sample_id, + sample_rate, + sequencing_kit, + sequencer_position, + sequencer_position_type, + software, + system_name, + system_type, + tracking_id))); return POD5_OK; } -inline bool check_read_data_struct(std::uint16_t struct_version, void const* row_data) { - static_assert(READ_BATCH_ROW_INFO_VERSION == READ_BATCH_ROW_INFO_VERSION_3, - "New versions must be explicitly loaded"); +inline bool check_read_data_struct(std::uint16_t struct_version, void const * row_data) +{ + static_assert( + READ_BATCH_ROW_INFO_VERSION == READ_BATCH_ROW_INFO_VERSION_3, + "New versions must be explicitly loaded"); if (!check_not_null(row_data)) { return false; @@ -828,25 +912,25 @@ inline bool check_read_data_struct(std::uint16_t struct_version, void const* row } if (struct_version == READ_BATCH_ROW_INFO_VERSION_3) { - auto const* typed_row_data = static_cast(row_data); - - if (!check_not_null(typed_row_data->read_id) || - !check_not_null(typed_row_data->read_number) || - !check_not_null(typed_row_data->start_sample) || - !check_not_null(typed_row_data->median_before) || - !check_not_null(typed_row_data->channel) || !check_not_null(typed_row_data->well) || - !check_not_null(typed_row_data->calibration_scale) || - !check_not_null(typed_row_data->calibration_offset) || - !check_not_null(typed_row_data->end_reason) || - !check_not_null(typed_row_data->end_reason_forced) || - !check_not_null(typed_row_data->run_info_id) || - !check_not_null(typed_row_data->num_minknow_events) || - !check_not_null(typed_row_data->tracked_scaling_scale) || - !check_not_null(typed_row_data->tracked_scaling_shift) || - !check_not_null(typed_row_data->predicted_scaling_scale) || - !check_not_null(typed_row_data->predicted_scaling_shift) || - !check_not_null(typed_row_data->num_reads_since_mux_change) || - !check_not_null(typed_row_data->time_since_mux_change)) { + auto const * typed_row_data = static_cast(row_data); + + if (!check_not_null(typed_row_data->read_id) || !check_not_null(typed_row_data->read_number) + || !check_not_null(typed_row_data->start_sample) + || !check_not_null(typed_row_data->median_before) + || !check_not_null(typed_row_data->channel) || !check_not_null(typed_row_data->well) + || !check_not_null(typed_row_data->calibration_scale) + || !check_not_null(typed_row_data->calibration_offset) + || !check_not_null(typed_row_data->end_reason) + || !check_not_null(typed_row_data->end_reason_forced) + || !check_not_null(typed_row_data->run_info_id) + || !check_not_null(typed_row_data->num_minknow_events) + || !check_not_null(typed_row_data->tracked_scaling_scale) + || !check_not_null(typed_row_data->tracked_scaling_shift) + || !check_not_null(typed_row_data->predicted_scaling_scale) + || !check_not_null(typed_row_data->predicted_scaling_shift) + || !check_not_null(typed_row_data->num_reads_since_mux_change) + || !check_not_null(typed_row_data->time_since_mux_change)) + { return false; } } @@ -854,21 +938,26 @@ inline bool check_read_data_struct(std::uint16_t struct_version, void const* row return true; } -inline bool load_struct_row_into_read_data(std::unique_ptr const& writer, - pod5::ReadData& read_data, - std::uint16_t struct_version, - void const* row_data, - std::uint32_t row_id) { - static_assert(READ_BATCH_ROW_INFO_VERSION == READ_BATCH_ROW_INFO_VERSION_3, - "New versions must be explicitly loaded"); +inline bool load_struct_row_into_read_data( + std::unique_ptr const & writer, + pod5::ReadData & read_data, + std::uint16_t struct_version, + void const * row_data, + std::uint32_t row_id) +{ + static_assert( + READ_BATCH_ROW_INFO_VERSION == READ_BATCH_ROW_INFO_VERSION_3, + "New versions must be explicitly loaded"); // Version 0-2 are no longer supported for writing. if (struct_version == READ_BATCH_ROW_INFO_VERSION_3) { - auto const* typed_row_data = static_cast(row_data); + auto const * typed_row_data = static_cast(row_data); boost::uuids::uuid read_id_uuid; - std::copy(typed_row_data->read_id[row_id], - typed_row_data->read_id[row_id] + sizeof(read_id_uuid), read_id_uuid.begin()); + std::copy( + typed_row_data->read_id[row_id], + typed_row_data->read_id[row_id] + sizeof(read_id_uuid), + read_id_uuid.begin()); pod5::ReadEndReason end_reason_internal = pod5::ReadEndReason::unknown; switch (typed_row_data->end_reason[row_id]) { @@ -891,8 +980,8 @@ inline bool load_struct_row_into_read_data(std::unique_ptr con end_reason_internal = pod5::ReadEndReason::signal_negative; break; default: - pod5_set_error(arrow::Status::Invalid( - "out of range end reason passed to pod5_add_end_reason")); + pod5_set_error( + arrow::Status::Invalid("out of range end reason passed to pod5_add_end_reason")); return false; } @@ -903,40 +992,42 @@ inline bool load_struct_row_into_read_data(std::unique_ptr con } read_data = pod5::ReadData{ - read_id_uuid, - typed_row_data->read_number[row_id], - typed_row_data->start_sample[row_id], - typed_row_data->channel[row_id], - typed_row_data->well[row_id], - typed_row_data->pore_type[row_id], - typed_row_data->calibration_offset[row_id], - typed_row_data->calibration_scale[row_id], - typed_row_data->median_before[row_id], - *end_reason_index, - typed_row_data->end_reason_forced[row_id] != 0, - typed_row_data->run_info_id[row_id], - typed_row_data->num_minknow_events[row_id], - typed_row_data->tracked_scaling_scale[row_id], - typed_row_data->tracked_scaling_shift[row_id], - typed_row_data->predicted_scaling_scale[row_id], - typed_row_data->predicted_scaling_shift[row_id], - typed_row_data->num_reads_since_mux_change[row_id], - typed_row_data->time_since_mux_change[row_id], + read_id_uuid, + typed_row_data->read_number[row_id], + typed_row_data->start_sample[row_id], + typed_row_data->channel[row_id], + typed_row_data->well[row_id], + typed_row_data->pore_type[row_id], + typed_row_data->calibration_offset[row_id], + typed_row_data->calibration_scale[row_id], + typed_row_data->median_before[row_id], + *end_reason_index, + typed_row_data->end_reason_forced[row_id] != 0, + typed_row_data->run_info_id[row_id], + typed_row_data->num_minknow_events[row_id], + typed_row_data->tracked_scaling_scale[row_id], + typed_row_data->tracked_scaling_shift[row_id], + typed_row_data->predicted_scaling_scale[row_id], + typed_row_data->predicted_scaling_shift[row_id], + typed_row_data->num_reads_since_mux_change[row_id], + typed_row_data->time_since_mux_change[row_id], }; } else { - pod5_set_error(arrow::Status::Invalid("Invalid writer struct version '", struct_version, - "' passed")); + pod5_set_error( + arrow::Status::Invalid("Invalid writer struct version '", struct_version, "' passed")); return false; } return true; }; -pod5_error_t pod5_add_reads_data(Pod5FileWriter_t* file, - uint32_t read_count, - uint16_t struct_version, - void const* row_data, - int16_t const** signal, - uint32_t const* signal_size) { +pod5_error_t pod5_add_reads_data( + Pod5FileWriter_t * file, + uint32_t read_count, + uint16_t struct_version, + void const * row_data, + int16_t const ** signal, + uint32_t const * signal_size) +{ pod5_reset_error(); if (!check_file_not_null(file) || !check_read_data_struct(struct_version, row_data)) { @@ -945,26 +1036,28 @@ pod5_error_t pod5_add_reads_data(Pod5FileWriter_t* file, for (std::uint32_t read = 0; read < read_count; ++read) { pod5::ReadData read_data; - if (!load_struct_row_into_read_data(file->writer, read_data, struct_version, row_data, - read)) { + if (!load_struct_row_into_read_data( + file->writer, read_data, struct_version, row_data, read)) { return g_pod5_error_no; } POD5_C_RETURN_NOT_OK(file->writer->add_complete_read( - read_data, gsl::make_span(signal[read], signal_size[read]))); + read_data, gsl::make_span(signal[read], signal_size[read]))); } return POD5_OK; } -pod5_error_t pod5_add_reads_data_pre_compressed(Pod5FileWriter_t* file, - uint32_t read_count, - uint16_t struct_version, - void const* row_data, - char const*** compressed_signal, - size_t const** compressed_signal_size, - uint32_t const** sample_counts, - size_t const* signal_chunk_count) { +pod5_error_t pod5_add_reads_data_pre_compressed( + Pod5FileWriter_t * file, + uint32_t read_count, + uint16_t struct_version, + void const * row_data, + char const *** compressed_signal, + size_t const ** compressed_signal_size, + uint32_t const ** sample_counts, + size_t const * signal_chunk_count) +{ pod5_reset_error(); if (!check_file_not_null(file) || !check_read_data_struct(struct_version, row_data)) { @@ -973,8 +1066,8 @@ pod5_error_t pod5_add_reads_data_pre_compressed(Pod5FileWriter_t* file, for (std::uint32_t read = 0; read < read_count; ++read) { pod5::ReadData read_data; - if (!load_struct_row_into_read_data(file->writer, read_data, struct_version, row_data, - read)) { + if (!load_struct_row_into_read_data( + file->writer, read_data, struct_version, row_data, read)) { return g_pod5_error_no; } @@ -986,43 +1079,51 @@ pod5_error_t pod5_add_reads_data_pre_compressed(Pod5FileWriter_t* file, auto sample_count = sample_counts[read][i]; total_sample_count += sample_count; POD5_C_ASSIGN_OR_RAISE( - auto row_id, - file->writer->add_pre_compressed_signal( - read_data.read_id, - gsl::make_span(signal, signal_size).as_span(), - sample_count)); + auto row_id, + file->writer->add_pre_compressed_signal( + read_data.read_id, + gsl::make_span(signal, signal_size).as_span(), + sample_count)); signal_rows.push_back(row_id); } - POD5_C_RETURN_NOT_OK(file->writer->add_complete_read(read_data, gsl::make_span(signal_rows), - total_sample_count)); + POD5_C_RETURN_NOT_OK(file->writer->add_complete_read( + read_data, gsl::make_span(signal_rows), total_sample_count)); } return POD5_OK; } -size_t pod5_vbz_compressed_signal_max_size(size_t sample_count) { +size_t pod5_vbz_compressed_signal_max_size(size_t sample_count) +{ pod5_reset_error(); return pod5::compressed_signal_max_size(sample_count); } -pod5_error_t pod5_vbz_compress_signal(int16_t const* signal, - size_t signal_size, - char* compressed_signal_out, - size_t* compressed_signal_size) { +pod5_error_t pod5_vbz_compress_signal( + int16_t const * signal, + size_t signal_size, + char * compressed_signal_out, + size_t * compressed_signal_size) +{ pod5_reset_error(); - if (!check_not_null(signal) || !check_output_pointer_not_null(compressed_signal_out) || - !check_output_pointer_not_null(compressed_signal_size)) { + if (!check_not_null(signal) || !check_output_pointer_not_null(compressed_signal_out) + || !check_output_pointer_not_null(compressed_signal_size)) + { return g_pod5_error_no; } - POD5_C_ASSIGN_OR_RAISE(auto buffer, pod5::compress_signal(gsl::make_span(signal, signal_size), - arrow::system_memory_pool())); + POD5_C_ASSIGN_OR_RAISE( + auto buffer, + pod5::compress_signal(gsl::make_span(signal, signal_size), arrow::system_memory_pool())); if ((std::size_t)buffer->size() > *compressed_signal_size) { - pod5_set_error(pod5::Status::Invalid("Compressed signal size (", buffer->size(), - ") is greater than provided buffer size (", - compressed_signal_size, ")")); + pod5_set_error(pod5::Status::Invalid( + "Compressed signal size (", + buffer->size(), + ") is greater than provided buffer size (", + compressed_signal_size, + ")")); return g_pod5_error_no; } @@ -1032,10 +1133,12 @@ pod5_error_t pod5_vbz_compress_signal(int16_t const* signal, return POD5_OK; } -pod5_error_t pod5_vbz_decompress_signal(char const* compressed_signal, - size_t compressed_signal_size, - size_t sample_count, - short* signal_out) { +pod5_error_t pod5_vbz_decompress_signal( + char const * compressed_signal, + size_t compressed_signal_size, + size_t sample_count, + short * signal_out) +{ pod5_reset_error(); if (!check_not_null(compressed_signal) || !check_output_pointer_not_null(signal_out)) { @@ -1043,21 +1146,22 @@ pod5_error_t pod5_vbz_decompress_signal(char const* compressed_signal, } auto const in_span = - gsl::make_span(compressed_signal, compressed_signal_size).as_span(); + gsl::make_span(compressed_signal, compressed_signal_size).as_span(); auto out_span = gsl::make_span(signal_out, sample_count); POD5_C_RETURN_NOT_OK(pod5::decompress_signal(in_span, arrow::system_memory_pool(), out_span)); return POD5_OK; } -pod5_error_t pod5_format_read_id(read_id_t const read_id, char* read_id_string) { +pod5_error_t pod5_format_read_id(read_id_t const read_id, char * read_id_string) +{ pod5_reset_error(); if (!check_not_null(read_id) || !check_output_pointer_not_null(read_id_string)) { return g_pod5_error_no; } - auto uuid_data = reinterpret_cast(read_id); + auto uuid_data = reinterpret_cast(read_id); std::string string_data = boost::uuids::to_string(*uuid_data); if (string_data.size() != 36) { pod5_set_error(pod5::Status::Invalid("Unexpected length of UUID")); diff --git a/c++/pod5_format/c_api.h b/c++/pod5_format/c_api.h index 4652f76..6ea90b2 100644 --- a/c++/pod5_format/c_api.h +++ b/c++/pod5_format/c_api.h @@ -43,7 +43,7 @@ typedef enum pod5_error pod5_error_t; POD5_FORMAT_EXPORT pod5_error_t pod5_get_error_no(); /// \brief Get the most recent error description string from all pod5 api's. /// \note The string's lifetime is internally managed, a caller should not free it. -POD5_FORMAT_EXPORT char const* pod5_get_error_string(); +POD5_FORMAT_EXPORT char const * pod5_get_error_string(); //--------------------------------------------------------------------------------------------------------------------- // Global state @@ -129,47 +129,47 @@ typedef struct ReadBatchRowInfoV3 ReadBatchRowInfo_t; // Array of read data: struct ReadBatchRowInfoArrayV3 { // The read id data, in binary form. - read_id_t const* read_id; + read_id_t const * read_id; // Read number for the read. - uint32_t const* read_number; + uint32_t const * read_number; // Start sample for the read. - uint64_t const* start_sample; + uint64_t const * start_sample; // Median before level. - float const* median_before; + float const * median_before; // Channel for the read. - uint16_t const* channel; + uint16_t const * channel; // Well for the read. - uint8_t const* well; + uint8_t const * well; // Pore type for the read. - int16_t const* pore_type; + int16_t const * pore_type; // Calibration offset type for the read. - float const* calibration_offset; + float const * calibration_offset; // Palibration type for the read. - float const* calibration_scale; + float const * calibration_scale; // End reason type for the read. - pod5_end_reason_t const* end_reason; + pod5_end_reason_t const * end_reason; // Was the end reason for the read forced (0 for false, 1 for true). - uint8_t const* end_reason_forced; + uint8_t const * end_reason_forced; // Run info type for the read. - int16_t const* run_info_id; + int16_t const * run_info_id; // Number of minknow events that the read contains - uint64_t const* num_minknow_events; + uint64_t const * num_minknow_events; // Scale/Shift for tracked read scaling values (based on previous reads) - float const* tracked_scaling_scale; - float const* tracked_scaling_shift; + float const * tracked_scaling_scale; + float const * tracked_scaling_shift; // Scale/Shift for predicted read scaling values (based on this read's raw signal) - float const* predicted_scaling_scale; - float const* predicted_scaling_shift; + float const * predicted_scaling_scale; + float const * predicted_scaling_shift; // How many reads have been selected prior to this read on the channel-well since it was made active. - uint32_t const* num_reads_since_mux_change; + uint32_t const * num_reads_since_mux_change; // How many seconds have passed since the channel-well was made active - float const* time_since_mux_change; + float const * time_since_mux_change; }; // Typedef for latest batch row info structure. @@ -191,10 +191,10 @@ typedef struct ReadBatchRowInfoArrayV3 ReadBatchRowInfoArray_t; /// \brief Open a file reader /// \param filename The filename of the pod5 file. -POD5_FORMAT_EXPORT Pod5FileReader_t* pod5_open_file(char const* filename); +POD5_FORMAT_EXPORT Pod5FileReader_t * pod5_open_file(char const * filename); /// \brief Close a file reader, releasing all memory held by the reader. -POD5_FORMAT_EXPORT pod5_error_t pod5_close_and_free_reader(Pod5FileReader_t* file); +POD5_FORMAT_EXPORT pod5_error_t pod5_close_and_free_reader(Pod5FileReader_t * file); struct FileInfo { read_id_t file_identifier; @@ -210,11 +210,12 @@ typedef struct FileInfo FileInfo_t; /// \brief Find the number of read batches in the file. /// \param[out] file The file to be queried. /// \param file_info The info read from the file. -POD5_FORMAT_EXPORT pod5_error_t pod5_get_file_info(Pod5FileReader_t* reader, FileInfo_t* file_info); +POD5_FORMAT_EXPORT pod5_error_t +pod5_get_file_info(Pod5FileReader_t * reader, FileInfo_t * file_info); struct EmbeddedFileData { // The file name to open - note this may not be the original file name, if the file has been migrated. - char const* file_name; + char const * file_name; size_t offset; size_t length; }; @@ -223,20 +224,20 @@ typedef struct EmbeddedFileData EmbeddedFileData_t; /// \brief Find the location of the read table data /// \param[out] file The file to be queried. /// \param file_data The output read table file data. -POD5_FORMAT_EXPORT pod5_error_t pod5_get_file_read_table_location(Pod5FileReader_t* reader, - EmbeddedFileData_t* file_data); +POD5_FORMAT_EXPORT pod5_error_t +pod5_get_file_read_table_location(Pod5FileReader_t * reader, EmbeddedFileData_t * file_data); /// \brief Find the location of the signal table data /// \param[out] file The file to be queried. /// \param file_data The output signal table file data. -POD5_FORMAT_EXPORT pod5_error_t pod5_get_file_signal_table_location(Pod5FileReader_t* reader, - EmbeddedFileData_t* file_data); +POD5_FORMAT_EXPORT pod5_error_t +pod5_get_file_signal_table_location(Pod5FileReader_t * reader, EmbeddedFileData_t * file_data); /// \brief Find the location of the run info table data /// \param[out] file The file to be queried. /// \param file_data The output signal table file data. POD5_FORMAT_EXPORT pod5_error_t -pod5_get_file_run_info_table_location(Pod5FileReader_t* reader, EmbeddedFileData_t* file_data); +pod5_get_file_run_info_table_location(Pod5FileReader_t * reader, EmbeddedFileData_t * file_data); /// \brief Plan the most efficient route through the data for the given read ids /// \param file The file to be queried. @@ -252,35 +253,36 @@ pod5_get_file_run_info_table_location(Pod5FileReader_t* reader, EmbeddedFileData /// [find_success_count] is the number of successful find steps in the result [steps]. /// Failed finds are all sorted to the back of the [steps] array, and are marked with an /// invalid batch and batch_row value. -POD5_FORMAT_EXPORT pod5_error_t pod5_plan_traversal(Pod5FileReader_t* reader, - uint8_t const* read_id_array, - size_t read_id_count, - uint32_t* batch_counts, - uint32_t* batch_rows, - size_t* find_success_count); +POD5_FORMAT_EXPORT pod5_error_t pod5_plan_traversal( + Pod5FileReader_t * reader, + uint8_t const * read_id_array, + size_t read_id_count, + uint32_t * batch_counts, + uint32_t * batch_rows, + size_t * find_success_count); /// \brief Find the number of read batches in the file. /// \param[out] count The number of read batches in the file /// \param reader The file reader to read from -POD5_FORMAT_EXPORT pod5_error_t pod5_get_read_batch_count(size_t* count, Pod5FileReader_t* reader); +POD5_FORMAT_EXPORT pod5_error_t +pod5_get_read_batch_count(size_t * count, Pod5FileReader_t * reader); /// \brief Get a read batch from the file. /// \param[out] batch The extracted batch. /// \param reader The file reader to read from /// \param index The index of the batch to read. /// \note Batches returned from this API must be freed using #pod5_free_read_batch -POD5_FORMAT_EXPORT pod5_error_t pod5_get_read_batch(Pod5ReadRecordBatch_t** batch, - Pod5FileReader_t* reader, - size_t index); +POD5_FORMAT_EXPORT pod5_error_t +pod5_get_read_batch(Pod5ReadRecordBatch_t ** batch, Pod5FileReader_t * reader, size_t index); /// \brief Release a read batch when it is not longer used. /// \param batch The batch to release. -POD5_FORMAT_EXPORT pod5_error_t pod5_free_read_batch(Pod5ReadRecordBatch_t* batch); +POD5_FORMAT_EXPORT pod5_error_t pod5_free_read_batch(Pod5ReadRecordBatch_t * batch); /// \brief Find the number of rows in a batch. /// \param batch The batch to query the number of rows for. -POD5_FORMAT_EXPORT pod5_error_t pod5_get_read_batch_row_count(size_t* count, - Pod5ReadRecordBatch_t*); +POD5_FORMAT_EXPORT pod5_error_t +pod5_get_read_batch_row_count(size_t * count, Pod5ReadRecordBatch_t *); /// \brief Find the info for a row in a read batch. /// \param batch The read batch to query. @@ -290,11 +292,12 @@ POD5_FORMAT_EXPORT pod5_error_t pod5_get_read_batch_row_count(size_t* count, /// \param[out] row_data The data for reading into, should be a pointer to ReadBatchRowInfo_t. /// \param[out] read_table_version The table version read from the file, will indicate which fields should be available. /// See READ_BATCH_ROW_INFO_VERSION and ReadBatchRowInfo_t above for corresponding fields. -POD5_FORMAT_EXPORT pod5_error_t pod5_get_read_batch_row_info_data(Pod5ReadRecordBatch_t* batch, - size_t row, - uint16_t struct_version, - void* row_data, - uint16_t* read_table_version); +POD5_FORMAT_EXPORT pod5_error_t pod5_get_read_batch_row_info_data( + Pod5ReadRecordBatch_t * batch, + size_t row, + uint16_t struct_version, + void * row_data, + uint16_t * read_table_version); /// \brief Find the info for a row in a read batch. /// \param batch The read batch to query. @@ -303,10 +306,11 @@ POD5_FORMAT_EXPORT pod5_error_t pod5_get_read_batch_row_info_data(Pod5ReadRecord /// \param[out] signal_row_indices The signal row indices read out of the read row. /// \note signal_row_indices_count Must equal signal_row_count returned from pod5_get_read_batch_row_info /// or an error is generated. -POD5_FORMAT_EXPORT pod5_error_t pod5_get_signal_row_indices(Pod5ReadRecordBatch_t* batch, - size_t row, - int64_t signal_row_indices_count, - uint64_t* signal_row_indices); +POD5_FORMAT_EXPORT pod5_error_t pod5_get_signal_row_indices( + Pod5ReadRecordBatch_t * batch, + size_t row, + int64_t signal_row_indices_count, + uint64_t * signal_row_indices); struct CalibrationExtraData { // The digitisation value used by the sequencer, equal to: @@ -323,36 +327,37 @@ typedef struct CalibrationExtraData CalibrationExtraData_t; /// \param row The read row index. /// \param[out] calibration_extra_data Output location for the calibration data. /// \note The values are computed from data held in the file, and written directly to the address provided, there is no need to release any data. -POD5_FORMAT_EXPORT pod5_error_t -pod5_get_calibration_extra_info(Pod5ReadRecordBatch_t* batch, - size_t row, - CalibrationExtraData_t* calibration_extra_data); +POD5_FORMAT_EXPORT pod5_error_t pod5_get_calibration_extra_info( + Pod5ReadRecordBatch_t * batch, + size_t row, + CalibrationExtraData_t * calibration_extra_data); struct KeyValueData { size_t size; - char const** keys; - char const** values; + char const ** keys; + char const ** values; }; + struct RunInfoDictData { - char const* acquisition_id; + char const * acquisition_id; int64_t acquisition_start_time_ms; int16_t adc_max; int16_t adc_min; struct KeyValueData context_tags; - char const* experiment_name; - char const* flow_cell_id; - char const* flow_cell_product_code; - char const* protocol_name; - char const* protocol_run_id; + char const * experiment_name; + char const * flow_cell_id; + char const * flow_cell_product_code; + char const * protocol_name; + char const * protocol_run_id; int64_t protocol_start_time_ms; - char const* sample_id; + char const * sample_id; uint16_t sample_rate; - char const* sequencing_kit; - char const* sequencer_position; - char const* sequencer_position_type; - char const* software; - char const* system_name; - char const* system_type; + char const * sequencing_kit; + char const * sequencer_position; + char const * sequencer_position_type; + char const * software; + char const * system_name; + char const * system_type; struct KeyValueData tracking_id; }; typedef struct RunInfoDictData RunInfoDictData_t; @@ -362,12 +367,13 @@ typedef struct RunInfoDictData RunInfoDictData_t; /// \param run_info The run info index to query from the passed batch. /// \param[out] run_info_data Output location for the run info data. /// \note The returned end_reason value should be released using pod5_release_calibration when it is no longer used. -POD5_FORMAT_EXPORT pod5_error_t pod5_get_run_info(Pod5ReadRecordBatch_t* batch, - int16_t run_info, - RunInfoDictData_t** run_info_data); +POD5_FORMAT_EXPORT pod5_error_t pod5_get_run_info( + Pod5ReadRecordBatch_t * batch, + int16_t run_info, + RunInfoDictData_t ** run_info_data); /// \brief Release a RunInfoDictData struct after use. -POD5_FORMAT_EXPORT pod5_error_t pod5_release_run_info(RunInfoDictData_t* run_info_data); +POD5_FORMAT_EXPORT pod5_error_t pod5_release_run_info(RunInfoDictData_t * run_info_data); /// \brief Find the end reason for a row in a read batch. /// \param batch The read batch to query. @@ -376,11 +382,12 @@ POD5_FORMAT_EXPORT pod5_error_t pod5_release_run_info(RunInfoDictData_t* run_inf /// \param[out] end_reason_string_value Output location for the string value for the end reason. /// \param[inout] end_reason_string_value_size Size of [end_reason_string_value], the number of characters written (including 1 for null character) is placed in this value on return. /// \note If the string input is not long enough POD5_ERROR_STRING_NOT_LONG_ENOUGH is returned. -POD5_FORMAT_EXPORT pod5_error_t pod5_get_end_reason(Pod5ReadRecordBatch_t* batch, - int16_t end_reason, - pod5_end_reason_t* end_reason_value, - char* end_reason_string_value, - size_t* end_reason_string_value_size); +POD5_FORMAT_EXPORT pod5_error_t pod5_get_end_reason( + Pod5ReadRecordBatch_t * batch, + int16_t end_reason, + pod5_end_reason_t * end_reason_value, + char * end_reason_string_value, + size_t * end_reason_string_value_size); /// \brief Find the pore type for a row in a read batch. /// \param batch The read batch to query. @@ -388,10 +395,11 @@ POD5_FORMAT_EXPORT pod5_error_t pod5_get_end_reason(Pod5ReadRecordBatch_t* batch /// \param[out] pore_type_string_value Output location for the string value for the pore type. /// \param[inout] pore_type_string_value_size Size of [pore_type_string_value], the number of characters written (including 1 for null character) is placed in this value on return. /// \note If the string input is not long enough POD5_ERROR_STRING_NOT_LONG_ENOUGH is returned. -POD5_FORMAT_EXPORT pod5_error_t pod5_get_pore_type(Pod5ReadRecordBatch_t* batch, - int16_t pore_type, - char* pore_type_string_value, - size_t* pore_type_string_value_size); +POD5_FORMAT_EXPORT pod5_error_t pod5_get_pore_type( + Pod5ReadRecordBatch_t * batch, + int16_t pore_type, + char * pore_type_string_value, + size_t * pore_type_string_value_size); struct SignalRowInfo { size_t batch_index; @@ -406,17 +414,18 @@ typedef struct SignalRowInfo SignalRowInfo_t; /// \param signal_rows_count The number of signal rows to query. /// \param signal_rows The signal rows to query. /// \param[out] signal_row_info Pointers to the output signal row information (must be an array of size signal_rows_count) -POD5_FORMAT_EXPORT pod5_error_t pod5_get_signal_row_info(Pod5FileReader_t* reader, - size_t signal_rows_count, - uint64_t* signal_rows, - SignalRowInfo_t** signal_row_info); +POD5_FORMAT_EXPORT pod5_error_t pod5_get_signal_row_info( + Pod5FileReader_t * reader, + size_t signal_rows_count, + uint64_t * signal_rows, + SignalRowInfo_t ** signal_row_info); /// \brief Release a list of signal row infos allocated by [pod5_get_signal_row_info]. /// \param signal_rows_count The number of signal rows to release. /// \param signal_row_info The signal row infos to release. /// \note Calls to pod5_free_signal_row_info must be 1:1 with [pod5_get_signal_row_info], you cannot free part of the returned data. -POD5_FORMAT_EXPORT pod5_error_t pod5_free_signal_row_info(size_t signal_rows_count, - SignalRowInfo_t** signal_row_info); +POD5_FORMAT_EXPORT pod5_error_t +pod5_free_signal_row_info(size_t signal_rows_count, SignalRowInfo_t ** signal_row_info); /// \brief Find the info for a signal row in a reader. /// \param reader The reader to query. @@ -425,20 +434,22 @@ POD5_FORMAT_EXPORT pod5_error_t pod5_free_signal_row_info(size_t signal_rows_cou /// \param[out] sample_data The output location for the queried samples. /// \note The signal data is allocated by the caller and should be released as appropriate by the caller. /// \todo MAJOR_VERSION Rename to include "chunk" or "row" or similar to indicate this gets only part of read signal. -POD5_FORMAT_EXPORT pod5_error_t pod5_get_signal(Pod5FileReader_t* reader, - SignalRowInfo_t* row_info, - size_t sample_count, - int16_t* sample_data); +POD5_FORMAT_EXPORT pod5_error_t pod5_get_signal( + Pod5FileReader_t * reader, + SignalRowInfo_t * row_info, + size_t sample_count, + int16_t * sample_data); /// \brief Find the sample count for a full read. /// \param reader The reader to query. /// \param batch The read batch to query. /// \param batch_row The read row to query data for. /// \param[out] sample_count The number of samples in the read - including all chunks of raw data. -POD5_FORMAT_EXPORT pod5_error_t pod5_get_read_complete_sample_count(Pod5FileReader_t* reader, - Pod5ReadRecordBatch_t* batch, - size_t batch_row, - size_t* sample_count); +POD5_FORMAT_EXPORT pod5_error_t pod5_get_read_complete_sample_count( + Pod5FileReader_t * reader, + Pod5ReadRecordBatch_t * batch, + size_t batch_row, + size_t * sample_count); /// \brief Find the signal for a full read. /// \param reader The reader to query. @@ -447,11 +458,12 @@ POD5_FORMAT_EXPORT pod5_error_t pod5_get_read_complete_sample_count(Pod5FileRead /// \param sample_count The number of samples allocated in [sample_data] (must equal the length of signal data in the queryied read row). /// \param[out] sample_data The output location for the queried samples. /// \note The signal data is allocated by pod5 and should be released as appropriate by the caller. -POD5_FORMAT_EXPORT pod5_error_t pod5_get_read_complete_signal(Pod5FileReader_t* reader, - Pod5ReadRecordBatch_t* batch, - size_t batch_row, - size_t sample_count, - int16_t* signal); +POD5_FORMAT_EXPORT pod5_error_t pod5_get_read_complete_signal( + Pod5FileReader_t * reader, + Pod5ReadRecordBatch_t * batch, + size_t batch_row, + size_t sample_count, + int16_t * signal); //--------------------------------------------------------------------------------------------------------------------- // Writing files @@ -487,20 +499,20 @@ typedef struct Pod5WriterOptions Pod5WriterOptions_t; /// \param filename The filename of the pod5 file. /// \param writer_name A descriptive string for the user software writing this file. /// \param options Options controlling how the file will be written. -POD5_FORMAT_EXPORT Pod5FileWriter_t* pod5_create_file(char const* filename, - char const* writer_name, - Pod5WriterOptions_t const* options); +POD5_FORMAT_EXPORT Pod5FileWriter_t * pod5_create_file( + char const * filename, + char const * writer_name, + Pod5WriterOptions_t const * options); /// \brief Close a file writer, releasing all memory held by the writer. -POD5_FORMAT_EXPORT pod5_error_t pod5_close_and_free_writer(Pod5FileWriter_t* file); +POD5_FORMAT_EXPORT pod5_error_t pod5_close_and_free_writer(Pod5FileWriter_t * file); /// \brief Add a new pore type to the file. /// \param[out] pore_index The index of the added pore. /// \param file The file to add the new pore type to. /// \param pore_type The pore type string for the pore. -POD5_FORMAT_EXPORT pod5_error_t pod5_add_pore(int16_t* pore_index, - Pod5FileWriter_t* file, - char const* pore_type); +POD5_FORMAT_EXPORT pod5_error_t +pod5_add_pore(int16_t * pore_index, Pod5FileWriter_t * file, char const * pore_type); /// \brief Add a new run info to the file, containing tracking information about a sequencing run. /// \param[out] run_info_index The index of the added run_info. @@ -529,32 +541,33 @@ POD5_FORMAT_EXPORT pod5_error_t pod5_add_pore(int16_t* pore_index, /// \param tracking_id_count Number of entries in the tracking id map. /// \param tracking_id_keys Array of strings used as keys into the tracking id map (must have tracking_id_count entries). /// \param tracking_id_values Array of strings used as values in the tracking id map (must have tracking_id_count entries). -POD5_FORMAT_EXPORT pod5_error_t pod5_add_run_info(int16_t* run_info_index, - Pod5FileWriter_t* file, - char const* acquisition_id, - int64_t acquisition_start_time_ms, - int16_t adc_max, - int16_t adc_min, - size_t context_tags_count, - char const** context_tags_keys, - char const** context_tags_values, - char const* experiment_name, - char const* flow_cell_id, - char const* flow_cell_product_code, - char const* protocol_name, - char const* protocol_run_id, - int64_t protocol_start_time_ms, - char const* sample_id, - uint16_t sample_rate, - char const* sequencing_kit, - char const* sequencer_position, - char const* sequencer_position_type, - char const* software, - char const* system_name, - char const* system_type, - size_t tracking_id_count, - char const** tracking_id_keys, - char const** tracking_id_values); +POD5_FORMAT_EXPORT pod5_error_t pod5_add_run_info( + int16_t * run_info_index, + Pod5FileWriter_t * file, + char const * acquisition_id, + int64_t acquisition_start_time_ms, + int16_t adc_max, + int16_t adc_min, + size_t context_tags_count, + char const ** context_tags_keys, + char const ** context_tags_values, + char const * experiment_name, + char const * flow_cell_id, + char const * flow_cell_product_code, + char const * protocol_name, + char const * protocol_run_id, + int64_t protocol_start_time_ms, + char const * sample_id, + uint16_t sample_rate, + char const * sequencing_kit, + char const * sequencer_position, + char const * sequencer_position_type, + char const * software, + char const * system_name, + char const * system_type, + size_t tracking_id_count, + char const ** tracking_id_keys, + char const ** tracking_id_values); /// \brief Add a read to the file. /// @@ -571,12 +584,13 @@ POD5_FORMAT_EXPORT pod5_error_t pod5_add_run_info(int16_t* run_info_index, /// \param signal The signal data for the reads. /// \param signal_size The number of samples in the signal data. /// This must be an array of length [read_count]. -POD5_FORMAT_EXPORT pod5_error_t pod5_add_reads_data(Pod5FileWriter_t* file, - uint32_t read_count, - uint16_t struct_version, - void const* row_data, - int16_t const** signal, - uint32_t const* signal_size); +POD5_FORMAT_EXPORT pod5_error_t pod5_add_reads_data( + Pod5FileWriter_t * file, + uint32_t read_count, + uint16_t struct_version, + void const * row_data, + int16_t const ** signal, + uint32_t const * signal_size); /// \brief Add a read to the file, with pre compressed signal chunk sections. /// @@ -605,15 +619,15 @@ POD5_FORMAT_EXPORT pod5_error_t pod5_add_reads_data(Pod5FileWriter_t* file, /// corresponding [compressed_signal] array, in samples (not bytes!). /// \param signal_chunk_count The number of sections of compressed signal. /// This must be an array of length [read_count]. -POD5_FORMAT_EXPORT pod5_error_t -pod5_add_reads_data_pre_compressed(Pod5FileWriter_t* file, - uint32_t read_count, - uint16_t struct_version, - void const* row_data, - char const*** compressed_signal, - size_t const** compressed_signal_size, - uint32_t const** sample_counts, - size_t const* signal_chunk_count); +POD5_FORMAT_EXPORT pod5_error_t pod5_add_reads_data_pre_compressed( + Pod5FileWriter_t * file, + uint32_t read_count, + uint16_t struct_version, + void const * row_data, + char const *** compressed_signal, + size_t const ** compressed_signal_size, + uint32_t const ** sample_counts, + size_t const * signal_chunk_count); /// \brief Find the max size of a compressed array of samples. /// \param sample_count The number of samples in the source signal. @@ -625,20 +639,22 @@ POD5_FORMAT_EXPORT size_t pod5_vbz_compressed_signal_max_size(size_t sample_coun /// \param signal_size The number of samples to compress. /// \param[out] compressed_signal_out The compressed signal. /// \param[inout] compressed_signal_size The number of compressed bytes, should be set to the size of compressed_signal_out on call. -POD5_FORMAT_EXPORT pod5_error_t pod5_vbz_compress_signal(int16_t const* signal, - size_t signal_size, - char* compressed_signal_out, - size_t* compressed_signal_size); +POD5_FORMAT_EXPORT pod5_error_t pod5_vbz_compress_signal( + int16_t const * signal, + size_t signal_size, + char * compressed_signal_out, + size_t * compressed_signal_size); /// \brief VBZ decompress an array of samples. /// \param compressed_signal The signal to compress. /// \param compressed_signal_size The number of compressed bytes, should be set to the size of compressed_signal_out on call. /// \param sample_count The number of samples to decompress. /// \param[out] signal_out The compressed signal. -POD5_FORMAT_EXPORT pod5_error_t pod5_vbz_decompress_signal(char const* compressed_signal, - size_t compressed_signal_size, - size_t sample_count, - short* signal_out); +POD5_FORMAT_EXPORT pod5_error_t pod5_vbz_decompress_signal( + char const * compressed_signal, + size_t compressed_signal_size, + size_t sample_count, + short * signal_out); //--------------------------------------------------------------------------------------------------------------------- // Global state @@ -647,7 +663,7 @@ POD5_FORMAT_EXPORT pod5_error_t pod5_vbz_decompress_signal(char const* compresse /// \brief Format a packed binary read id as a readable read id string: /// \param read_id A 16 byte binary formatted UUID. /// \param[out] read_id_string Output string containing the string formatted UUID (expects a string of at least 37 bytes, one null byte is written.) -POD5_FORMAT_EXPORT pod5_error_t pod5_format_read_id(read_id_t const read_id, char* read_id_string); +POD5_FORMAT_EXPORT pod5_error_t pod5_format_read_id(read_id_t const read_id, char * read_id_string); #ifdef __cplusplus } diff --git a/c++/pod5_format/dictionary_writer.h b/c++/pod5_format/dictionary_writer.h index 4372ae7..ac3d51e 100644 --- a/c++/pod5_format/dictionary_writer.h +++ b/c++/pod5_format/dictionary_writer.h @@ -14,7 +14,7 @@ class POD5_FORMAT_EXPORT DictionaryWriter { virtual ~DictionaryWriter() = default; pod5::Result> build_dictionary_array( - std::shared_ptr const& indices); + std::shared_ptr const & indices); virtual pod5::Result> get_value_array() = 0; virtual std::size_t item_count() = 0; }; diff --git a/c++/pod5_format/errors.cpp b/c++/pod5_format/errors.cpp index ab74d0c..3c1b5f5 100644 --- a/c++/pod5_format/errors.cpp +++ b/c++/pod5_format/errors.cpp @@ -7,8 +7,11 @@ class ErrorCategory : public std::error_category { // Work around AppleClang issue: // https://stackoverflow.com/questions/7411515/why-does-c-require-a-user-provided-default-constructor-to-default-construct-a ErrorCategory() {} - const char *name() const noexcept override { return "usb"; } - std::string message(int code) const override { + + char const * name() const noexcept override { return "usb"; } + + std::string message(int code) const override + { if (code == 0) { return "Success"; } @@ -26,7 +29,8 @@ class ErrorCategory : public std::error_category { return "unknown error"; } - std::error_condition default_error_condition(int code) const noexcept override { + std::error_condition default_error_condition(int code) const noexcept override + { if (code == 0) { return std::error_condition(0, std::generic_category()); } @@ -45,7 +49,8 @@ class ErrorCategory : public std::error_category { } }; -const std::error_category &error_category() { +std::error_category const & error_category() +{ static const ErrorCategory category; return category; } diff --git a/c++/pod5_format/errors.h b/c++/pod5_format/errors.h index c7c7285..fb55623 100644 --- a/c++/pod5_format/errors.h +++ b/c++/pod5_format/errors.h @@ -19,9 +19,10 @@ enum class Errors : int { failed_to_write_record_batch = 3, }; -POD5_FORMAT_EXPORT std::error_category const& error_category(); +POD5_FORMAT_EXPORT std::error_category const & error_category(); -inline std::error_code make_error_code(Errors result) { +inline std::error_code make_error_code(Errors result) +{ return std::error_code(static_cast(result), error_category()); } diff --git a/c++/pod5_format/expandable_buffer.h b/c++/pod5_format/expandable_buffer.h index ae2f0a5..7a14004 100644 --- a/c++/pod5_format/expandable_buffer.h +++ b/c++/pod5_format/expandable_buffer.h @@ -11,24 +11,28 @@ class ExpandableBuffer { public: static constexpr int EXPANSION_FACTOR = 2; - ExpandableBuffer(arrow::MemoryPool* pool = nullptr) { m_pool = pool; } + ExpandableBuffer(arrow::MemoryPool * pool = nullptr) { m_pool = pool; } - arrow::Status init_buffer(arrow::MemoryPool* pool) { + arrow::Status init_buffer(arrow::MemoryPool * pool) + { m_pool = pool; return clear(); } - std::size_t size() const { + std::size_t size() const + { if (!m_buffer) { return 0; } return m_buffer->size() / sizeof(T); } - std::uint8_t* mutable_data() { return m_buffer->mutable_data(); } + + std::uint8_t * mutable_data() { return m_buffer->mutable_data(); } std::shared_ptr get_buffer() const { return m_buffer; } - arrow::Status clear() { + arrow::Status clear() + { if (!m_buffer || m_buffer.use_count() > 1) { ARROW_ASSIGN_OR_RAISE(m_buffer, arrow::AllocateResizableBuffer(0, m_pool)); return arrow::Status::OK(); @@ -37,30 +41,35 @@ class ExpandableBuffer { } } - gsl::span get_data_span() const { + gsl::span get_data_span() const + { assert(m_buffer); return gsl::make_span(m_buffer->data(), m_buffer->size()).template as_span(); } - arrow::Status append(T const& new_value) { + arrow::Status append(T const & new_value) + { auto const bytes_span = - gsl::make_span(&new_value, 1).template as_span(); + gsl::make_span(&new_value, 1).template as_span(); return append_bytes(bytes_span); } - arrow::Status append_array(gsl::span const& new_value_span) { + arrow::Status append_array(gsl::span const & new_value_span) + { auto const bytes_span = new_value_span.template as_span(); return append_bytes(bytes_span); } - arrow::Status resize(std::int64_t new_size) { + arrow::Status resize(std::int64_t new_size) + { ARROW_RETURN_NOT_OK(reserve(new_size)); return m_buffer->Resize(new_size); } - arrow::Status reserve(std::int64_t new_capacity) { + arrow::Status reserve(std::int64_t new_capacity) + { assert(m_buffer); auto const old_size = m_buffer->size(); if (m_buffer.use_count() > 1) { @@ -78,11 +87,12 @@ class ExpandableBuffer { } private: - arrow::Status append_bytes(gsl::span const& bytes_span) { + arrow::Status append_bytes(gsl::span const & bytes_span) + { auto old_size = 0; if (!m_buffer) { - ARROW_ASSIGN_OR_RAISE(m_buffer, - arrow::AllocateResizableBuffer(bytes_span.size(), m_pool)); + ARROW_ASSIGN_OR_RAISE( + m_buffer, arrow::AllocateResizableBuffer(bytes_span.size(), m_pool)); } else { old_size = m_buffer->size(); } @@ -95,7 +105,7 @@ class ExpandableBuffer { } std::shared_ptr m_buffer; - arrow::MemoryPool* m_pool = nullptr; + arrow::MemoryPool * m_pool = nullptr; }; } // namespace pod5 diff --git a/c++/pod5_format/file_reader.cpp b/c++/pod5_format/file_reader.cpp index 05f12c2..10f315a 100644 --- a/c++/pod5_format/file_reader.cpp +++ b/c++/pod5_format/file_reader.cpp @@ -15,87 +15,108 @@ namespace pod5 { FileReaderOptions::FileReaderOptions() : m_memory_pool(arrow::default_memory_pool()) {} -inline FileLocation make_file_locaton(combined_file_utils::ParsedFileInfo const& parsed_file_info) { - return FileLocation{parsed_file_info.file_path, std::size_t(parsed_file_info.file_start_offset), - std::size_t(parsed_file_info.file_length)}; +inline FileLocation make_file_locaton(combined_file_utils::ParsedFileInfo const & parsed_file_info) +{ + return FileLocation{ + parsed_file_info.file_path, + std::size_t(parsed_file_info.file_start_offset), + std::size_t(parsed_file_info.file_length)}; } class FileReaderImpl : public FileReader { public: - FileReaderImpl(Version const& file_version_pre_migration, - MigrationResult&& migration_result, - RunInfoTableReader&& run_info_table_reader, - ReadTableReader&& read_table_reader, - SignalTableReader&& signal_table_reader) - : m_file_version_pre_migration(file_version_pre_migration), - m_migration_result(std::move(migration_result)), - m_run_info_table_location( - make_file_locaton(m_migration_result.footer().run_info_table)), - m_read_table_location(make_file_locaton(m_migration_result.footer().reads_table)), - m_signal_table_location(make_file_locaton(m_migration_result.footer().signal_table)), - m_run_info_table_reader(std::move(run_info_table_reader)), - m_read_table_reader(std::move(read_table_reader)), - m_signal_table_reader(std::move(signal_table_reader)) {} - - SchemaMetadataDescription schema_metadata() const override { + FileReaderImpl( + Version const & file_version_pre_migration, + MigrationResult && migration_result, + RunInfoTableReader && run_info_table_reader, + ReadTableReader && read_table_reader, + SignalTableReader && signal_table_reader) + : m_file_version_pre_migration(file_version_pre_migration) + , m_migration_result(std::move(migration_result)) + , m_run_info_table_location(make_file_locaton(m_migration_result.footer().run_info_table)) + , m_read_table_location(make_file_locaton(m_migration_result.footer().reads_table)) + , m_signal_table_location(make_file_locaton(m_migration_result.footer().signal_table)) + , m_run_info_table_reader(std::move(run_info_table_reader)) + , m_read_table_reader(std::move(read_table_reader)) + , m_signal_table_reader(std::move(signal_table_reader)) + { + } + + SchemaMetadataDescription schema_metadata() const override + { return m_read_table_reader.schema_metadata(); } - Result read_read_record_batch(std::size_t i) const override { + Result read_read_record_batch(std::size_t i) const override + { return m_read_table_reader.read_record_batch(i); } - std::size_t num_read_record_batches() const override { + std::size_t num_read_record_batches() const override + { return m_read_table_reader.num_record_batches(); } - Result search_for_read_ids(ReadIdSearchInput const& search_input, - gsl::span const& batch_counts, - gsl::span const& batch_rows) override { + Result search_for_read_ids( + ReadIdSearchInput const & search_input, + gsl::span const & batch_counts, + gsl::span const & batch_rows) override + { return m_read_table_reader.search_for_read_ids(search_input, batch_counts, batch_rows); } - Result read_signal_record_batch(std::size_t i) const override { + Result read_signal_record_batch(std::size_t i) const override + { return m_signal_table_reader.read_record_batch(i); } - std::size_t num_signal_record_batches() const override { + std::size_t num_signal_record_batches() const override + { return m_signal_table_reader.num_record_batches(); } - Result signal_batch_for_row_id(std::size_t row, - std::size_t* batch_row) const override { + Result signal_batch_for_row_id(std::size_t row, std::size_t * batch_row) + const override + { return m_signal_table_reader.signal_batch_for_row_id(row, batch_row); } Result extract_sample_count( - gsl::span const& row_indices) const override { + gsl::span const & row_indices) const override + { return m_signal_table_reader.extract_sample_count(row_indices); } - Status extract_samples(gsl::span const& row_indices, - gsl::span const& output_samples) const override { + Status extract_samples( + gsl::span const & row_indices, + gsl::span const & output_samples) const override + { return m_signal_table_reader.extract_samples(row_indices, output_samples); } Result>> extract_samples_inplace( - gsl::span const& row_indices, - std::vector& sample_count) const override { + gsl::span const & row_indices, + std::vector & sample_count) const override + { return m_signal_table_reader.extract_samples_inplace(row_indices, sample_count); } - FileLocation const& run_info_table_location() const override { + FileLocation const & run_info_table_location() const override + { return m_run_info_table_location; } - FileLocation const& read_table_location() const override { return m_read_table_location; } - FileLocation const& signal_table_location() const override { return m_signal_table_location; } + + FileLocation const & read_table_location() const override { return m_read_table_location; } + + FileLocation const & signal_table_location() const override { return m_signal_table_location; } Version file_version_pre_migration() const override { return m_file_version_pre_migration; } SignalType signal_type() const override { return m_signal_table_reader.signal_type(); } Result> find_run_info( - std::string const& acquisition_id) const override { + std::string const & acquisition_id) const override + { return m_run_info_table_reader.find_run_info(acquisition_id); } @@ -110,52 +131,60 @@ class FileReaderImpl : public FileReader { SignalTableReader m_signal_table_reader; }; -pod5::Result> open_file_reader(std::string const& path, - FileReaderOptions const& options) { +pod5::Result> open_file_reader( + std::string const & path, + FileReaderOptions const & options) +{ auto pool = options.memory_pool(); if (!pool) { return Status::Invalid("Invalid memory pool specified for file writer"); } - ARROW_ASSIGN_OR_RAISE(auto file, - arrow::io::MemoryMappedFile::Open(path, arrow::io::FileMode::READ)); + ARROW_ASSIGN_OR_RAISE( + auto file, arrow::io::MemoryMappedFile::Open(path, arrow::io::FileMode::READ)); - ARROW_ASSIGN_OR_RAISE(auto original_footer_metadata, - combined_file_utils::read_footer(path, file)); + ARROW_ASSIGN_OR_RAISE( + auto original_footer_metadata, combined_file_utils::read_footer(path, file)); - ARROW_ASSIGN_OR_RAISE(auto const original_writer_version, - parse_version_number(original_footer_metadata.writer_pod5_version)); ARROW_ASSIGN_OR_RAISE( - auto migration_result, - migrate_if_required(original_writer_version, original_footer_metadata, file, pool)); + auto const original_writer_version, + parse_version_number(original_footer_metadata.writer_pod5_version)); + ARROW_ASSIGN_OR_RAISE( + auto migration_result, + migrate_if_required(original_writer_version, original_footer_metadata, file, pool)); // Files are written standalone, and so needs to be treated with a file offset - it wants to seek around as if the reads file is standalone: - ARROW_ASSIGN_OR_RAISE(auto run_info_sub_file, - open_sub_file(migration_result.footer().run_info_table)); - ARROW_ASSIGN_OR_RAISE(auto run_info_table_reader, - make_run_info_table_reader(run_info_sub_file, pool)); + ARROW_ASSIGN_OR_RAISE( + auto run_info_sub_file, open_sub_file(migration_result.footer().run_info_table)); + ARROW_ASSIGN_OR_RAISE( + auto run_info_table_reader, make_run_info_table_reader(run_info_sub_file, pool)); - ARROW_ASSIGN_OR_RAISE(auto reads_sub_file, - open_sub_file(migration_result.footer().reads_table)); + ARROW_ASSIGN_OR_RAISE( + auto reads_sub_file, open_sub_file(migration_result.footer().reads_table)); ARROW_ASSIGN_OR_RAISE(auto read_table_reader, make_read_table_reader(reads_sub_file, pool)); - ARROW_ASSIGN_OR_RAISE(auto signal_sub_file, - open_sub_file(migration_result.footer().signal_table)); - ARROW_ASSIGN_OR_RAISE(auto signal_table_reader, - make_signal_table_reader(signal_sub_file, pool)); + ARROW_ASSIGN_OR_RAISE( + auto signal_sub_file, open_sub_file(migration_result.footer().signal_table)); + ARROW_ASSIGN_OR_RAISE( + auto signal_table_reader, make_signal_table_reader(signal_sub_file, pool)); auto signal_metadata = signal_table_reader.schema_metadata(); auto reads_metadata = read_table_reader.schema_metadata(); if (signal_metadata.file_identifier != reads_metadata.file_identifier) { - return Status::Invalid("Invalid read and signal file pair signal identifier: ", - signal_metadata.file_identifier, - ", reads identifier: ", reads_metadata.file_identifier); + return Status::Invalid( + "Invalid read and signal file pair signal identifier: ", + signal_metadata.file_identifier, + ", reads identifier: ", + reads_metadata.file_identifier); } return std::make_shared( - original_writer_version, std::move(migration_result), std::move(run_info_table_reader), - std::move(read_table_reader), std::move(signal_table_reader)); + original_writer_version, + std::move(migration_result), + std::move(run_info_table_reader), + std::move(read_table_reader), + std::move(signal_table_reader)); } } // namespace pod5 diff --git a/c++/pod5_format/file_reader.h b/c++/pod5_format/file_reader.h index e26bb53..0a1d4fd 100644 --- a/c++/pod5_format/file_reader.h +++ b/c++/pod5_format/file_reader.h @@ -23,17 +23,22 @@ class POD5_FORMAT_EXPORT FileReaderOptions { public: FileReaderOptions(); - void memory_pool(arrow::MemoryPool* memory_pool) { m_memory_pool = memory_pool; } - arrow::MemoryPool* memory_pool() const { return m_memory_pool; } + void memory_pool(arrow::MemoryPool * memory_pool) { m_memory_pool = memory_pool; } + + arrow::MemoryPool * memory_pool() const { return m_memory_pool; } private: - arrow::MemoryPool* m_memory_pool; + arrow::MemoryPool * m_memory_pool; }; class POD5_FORMAT_EXPORT FileLocation { public: - FileLocation(std::string const& file_path_, std::size_t offset_, std::size_t size_) - : file_path(file_path_), offset(offset_), size(size_) {} + FileLocation(std::string const & file_path_, std::size_t offset_, std::size_t size_) + : file_path(file_path_) + , offset(offset_) + , size(size_) + { + } std::string file_path; std::size_t offset; @@ -53,47 +58,49 @@ class POD5_FORMAT_EXPORT FileReader { virtual Result read_read_record_batch(std::size_t i) const = 0; virtual std::size_t num_read_record_batches() const = 0; - virtual Result search_for_read_ids(ReadIdSearchInput const& search_input, - gsl::span const& batch_counts, - gsl::span const& batch_rows) = 0; + virtual Result search_for_read_ids( + ReadIdSearchInput const & search_input, + gsl::span const & batch_counts, + gsl::span const & batch_rows) = 0; virtual Result read_signal_record_batch(std::size_t i) const = 0; virtual std::size_t num_signal_record_batches() const = 0; - virtual Result signal_batch_for_row_id(std::size_t row, - std::size_t* batch_row) const = 0; + virtual Result signal_batch_for_row_id(std::size_t row, std::size_t * batch_row) + const = 0; /// \brief Find the number of samples in a given list of rows. /// \param row_indices The rows to query for sample ount. /// \returns The sum of all sample counts on input rows. virtual Result extract_sample_count( - gsl::span const& row_indices) const = 0; + gsl::span const & row_indices) const = 0; /// \brief Extract the samples for a list of rows. /// \param row_indices The rows to query for samples. /// \param output_samples The output samples from the rows. - virtual Status extract_samples(gsl::span const& row_indices, - gsl::span const& output_samples) const = 0; + virtual Status extract_samples( + gsl::span const & row_indices, + gsl::span const & output_samples) const = 0; /// \brief Extract the samples as written in the arrow table for a list of rows. /// \param row_indices The rows to query for samples. /// \param output_samples The output samples from the rows. virtual Result>> extract_samples_inplace( - gsl::span const& row_indices, - std::vector& sample_count) const = 0; + gsl::span const & row_indices, + std::vector & sample_count) const = 0; - virtual FileLocation const& run_info_table_location() const = 0; - virtual FileLocation const& read_table_location() const = 0; - virtual FileLocation const& signal_table_location() const = 0; + virtual FileLocation const & run_info_table_location() const = 0; + virtual FileLocation const & read_table_location() const = 0; + virtual FileLocation const & signal_table_location() const = 0; virtual Version file_version_pre_migration() const = 0; virtual SignalType signal_type() const = 0; virtual Result> find_run_info( - std::string const& acquisition_id) const = 0; + std::string const & acquisition_id) const = 0; }; POD5_FORMAT_EXPORT pod5::Result> open_file_reader( - std::string const& path, - FileReaderOptions const& options = {}); + std::string const & path, + FileReaderOptions const & options = {}); } // namespace pod5 diff --git a/c++/pod5_format/file_updater.cpp b/c++/pod5_format/file_updater.cpp index 050e289..9752e27 100644 --- a/c++/pod5_format/file_updater.cpp +++ b/c++/pod5_format/file_updater.cpp @@ -9,9 +9,11 @@ namespace pod5 { -pod5::Status update_file(arrow::MemoryPool* pool, - std::shared_ptr const& source, - std::string destination) { +pod5::Status update_file( + arrow::MemoryPool * pool, + std::shared_ptr const & source, + std::string destination) +{ ARROW_ASSIGN_OR_RAISE(auto main_file, arrow::io::FileOutputStream::Open(destination, false)); auto uuid_gen = boost::uuids::random_generator_mt19937(); @@ -23,25 +25,39 @@ pod5::Status update_file(arrow::MemoryPool* pool, ARROW_RETURN_NOT_OK(combined_file_utils::write_combined_header(main_file, section_marker)); ARROW_ASSIGN_OR_RAISE( - auto signal_info_table, - combined_file_utils::write_file_and_marker( - pool, main_file, source->signal_table_location(), - combined_file_utils::SubFileCleanup::LeaveOrignalFile, section_marker)); + auto signal_info_table, + combined_file_utils::write_file_and_marker( + pool, + main_file, + source->signal_table_location(), + combined_file_utils::SubFileCleanup::LeaveOrignalFile, + section_marker)); ARROW_ASSIGN_OR_RAISE( - auto run_info_info_table, - combined_file_utils::write_file_and_marker( - pool, main_file, source->run_info_table_location(), - combined_file_utils::SubFileCleanup::LeaveOrignalFile, section_marker)); + auto run_info_info_table, + combined_file_utils::write_file_and_marker( + pool, + main_file, + source->run_info_table_location(), + combined_file_utils::SubFileCleanup::LeaveOrignalFile, + section_marker)); ARROW_ASSIGN_OR_RAISE( - auto reads_info_table, - combined_file_utils::write_file_and_marker( - pool, main_file, source->read_table_location(), - combined_file_utils::SubFileCleanup::LeaveOrignalFile, section_marker)); + auto reads_info_table, + combined_file_utils::write_file_and_marker( + pool, + main_file, + source->read_table_location(), + combined_file_utils::SubFileCleanup::LeaveOrignalFile, + section_marker)); // Write full file footer: ARROW_RETURN_NOT_OK(combined_file_utils::write_footer( - main_file, section_marker, metadata.file_identifier, metadata.writing_software, - signal_info_table, run_info_info_table, reads_info_table)); + main_file, + section_marker, + metadata.file_identifier, + metadata.writing_software, + signal_info_table, + run_info_info_table, + reads_info_table)); return main_file->Close(); } diff --git a/c++/pod5_format/file_updater.h b/c++/pod5_format/file_updater.h index 982ff48..f22f465 100644 --- a/c++/pod5_format/file_updater.h +++ b/c++/pod5_format/file_updater.h @@ -16,8 +16,9 @@ class FileReader; /// \param source The source file data to write updated. /// \param destination The destination path to write the data to. /// \note The destination path should not be the same file that was opened for input. -pod5::Status update_file(arrow::MemoryPool* pool, - std::shared_ptr const& source, - std::string destination); +pod5::Status update_file( + arrow::MemoryPool * pool, + std::shared_ptr const & source, + std::string destination); } // namespace pod5 diff --git a/c++/pod5_format/file_writer.cpp b/c++/pod5_format/file_writer.cpp index 85cb8e6..5f248be 100644 --- a/c++/pod5_format/file_writer.cpp +++ b/c++/pod5_format/file_writer.cpp @@ -20,81 +20,95 @@ namespace pod5 { FileWriterOptions::FileWriterOptions() - : m_max_signal_chunk_size(DEFAULT_SIGNAL_CHUNK_SIZE), - m_memory_pool(arrow::default_memory_pool()), - m_signal_type(DEFAULT_SIGNAL_TYPE), - m_signal_table_batch_size(DEFAULT_SIGNAL_TABLE_BATCH_SIZE), - m_read_table_batch_size(DEFAULT_READ_TABLE_BATCH_SIZE), - m_run_info_table_batch_size(DEFAULT_RUN_INFO_TABLE_BATCH_SIZE) {} +: m_max_signal_chunk_size(DEFAULT_SIGNAL_CHUNK_SIZE) +, m_memory_pool(arrow::default_memory_pool()) +, m_signal_type(DEFAULT_SIGNAL_TYPE) +, m_signal_table_batch_size(DEFAULT_SIGNAL_TABLE_BATCH_SIZE) +, m_read_table_batch_size(DEFAULT_READ_TABLE_BATCH_SIZE) +, m_run_info_table_batch_size(DEFAULT_RUN_INFO_TABLE_BATCH_SIZE) +{ +} class FileWriterImpl { public: class WriterTypeImpl; + struct DictionaryWriters { std::shared_ptr end_reason_writer; std::shared_ptr pore_writer; std::shared_ptr run_info_writer; }; - FileWriterImpl(DictionaryWriters&& read_table_dict_writers, - RunInfoTableWriter&& run_info_table_writer, - ReadTableWriter&& read_table_writer, - SignalTableWriter&& signal_table_writer, - std::uint32_t signal_chunk_size, - arrow::MemoryPool* pool) - : m_read_table_dict_writers(std::move(read_table_dict_writers)), - m_run_info_table_writer(std::move(run_info_table_writer)), - m_read_table_writer(std::move(read_table_writer)), - m_signal_table_writer(std::move(signal_table_writer)), - m_signal_chunk_size(signal_chunk_size), - m_pool(pool) {} + FileWriterImpl( + DictionaryWriters && read_table_dict_writers, + RunInfoTableWriter && run_info_table_writer, + ReadTableWriter && read_table_writer, + SignalTableWriter && signal_table_writer, + std::uint32_t signal_chunk_size, + arrow::MemoryPool * pool) + : m_read_table_dict_writers(std::move(read_table_dict_writers)) + , m_run_info_table_writer(std::move(run_info_table_writer)) + , m_read_table_writer(std::move(read_table_writer)) + , m_signal_table_writer(std::move(signal_table_writer)) + , m_signal_chunk_size(signal_chunk_size) + , m_pool(pool) + { + } virtual ~FileWriterImpl() = default; - pod5::Result lookup_end_reason(ReadEndReason end_reason) { + pod5::Result lookup_end_reason(ReadEndReason end_reason) + { return m_read_table_dict_writers.end_reason_writer->lookup(end_reason); } - pod5::Result add_pore_type(std::string const& pore_type_data) { + pod5::Result add_pore_type(std::string const & pore_type_data) + { return m_read_table_dict_writers.pore_writer->add(pore_type_data); } - pod5::Result add_run_info(RunInfoData const& run_info_data) { + pod5::Result add_run_info(RunInfoData const & run_info_data) + { ARROW_RETURN_NOT_OK(m_run_info_table_writer->add_run_info(run_info_data)); return m_read_table_dict_writers.run_info_writer->add(run_info_data.acquisition_id); } - pod5::Status add_complete_read(ReadData const& read_data, - gsl::span const& signal) { + pod5::Status add_complete_read( + ReadData const & read_data, + gsl::span const & signal) + { if (!m_signal_table_writer || !m_read_table_writer) { return arrow::Status::Invalid("File writer closed, cannot write further data"); } - ARROW_ASSIGN_OR_RAISE(std::vector signal_rows, - add_signal(read_data.read_id, signal)); + ARROW_ASSIGN_OR_RAISE( + std::vector signal_rows, add_signal(read_data.read_id, signal)); // Write read data and signal row entries: auto read_table_row = m_read_table_writer->add_read( - read_data, gsl::make_span(signal_rows.data(), signal_rows.size()), signal.size()); + read_data, gsl::make_span(signal_rows.data(), signal_rows.size()), signal.size()); return read_table_row.status(); } - pod5::Status add_complete_read(ReadData const& read_data, - gsl::span const& signal_rows, - std::uint64_t signal_duration) { + pod5::Status add_complete_read( + ReadData const & read_data, + gsl::span const & signal_rows, + std::uint64_t signal_duration) + { if (!m_signal_table_writer || !m_read_table_writer) { return arrow::Status::Invalid("File writer closed, cannot write further data"); } // Write read data and signal row entries: auto read_table_row = - m_read_table_writer->add_read(read_data, signal_rows, signal_duration); + m_read_table_writer->add_read(read_data, signal_rows, signal_duration); return read_table_row.status(); } pod5::Result> add_signal( - boost::uuids::uuid const& read_id, - gsl::span const& signal) { + boost::uuids::uuid const & read_id, + gsl::span const & signal) + { if (!m_signal_table_writer || !m_read_table_writer) { return arrow::Status::Invalid("File writer closed, cannot write further data"); } @@ -106,46 +120,52 @@ class FileWriterImpl { for (std::size_t chunk_start = 0; chunk_start < signal.size(); chunk_start += m_signal_chunk_size) { std::size_t chunk_size = - std::min(signal.size() - chunk_start, m_signal_chunk_size); + std::min(signal.size() - chunk_start, m_signal_chunk_size); auto const chunk_span = signal.subspan(chunk_start, chunk_size); - ARROW_ASSIGN_OR_RAISE(auto row_index, - m_signal_table_writer->add_signal(read_id, chunk_span)); + ARROW_ASSIGN_OR_RAISE( + auto row_index, m_signal_table_writer->add_signal(read_id, chunk_span)); signal_rows.push_back(row_index); } return signal_rows; } pod5::Result add_pre_compressed_signal( - boost::uuids::uuid const& read_id, - gsl::span const& signal_bytes, - std::uint32_t sample_count) { + boost::uuids::uuid const & read_id, + gsl::span const & signal_bytes, + std::uint32_t sample_count) + { if (!m_signal_table_writer || !m_read_table_writer) { return arrow::Status::Invalid("File writer closed, cannot write further data"); } - return m_signal_table_writer->add_pre_compressed_signal(read_id, signal_bytes, - sample_count); + return m_signal_table_writer->add_pre_compressed_signal( + read_id, signal_bytes, sample_count); } SignalType signal_type() const { return m_signal_table_writer->signal_type(); } - pod5::Status close_run_info_table_writer() { + pod5::Status close_run_info_table_writer() + { if (m_run_info_table_writer) { ARROW_RETURN_NOT_OK(m_run_info_table_writer->close()); m_run_info_table_writer = boost::none; } return pod5::Status::OK(); } - pod5::Status close_read_table_writer() { + + pod5::Status close_read_table_writer() + { if (m_read_table_writer) { ARROW_RETURN_NOT_OK(m_read_table_writer->close()); m_read_table_writer = boost::none; } return pod5::Status::OK(); } - pod5::Status close_signal_table_writer() { + + pod5::Status close_signal_table_writer() + { if (m_signal_table_writer) { ARROW_RETURN_NOT_OK(m_signal_table_writer->close()); m_signal_table_writer = boost::none; @@ -153,18 +173,20 @@ class FileWriterImpl { return pod5::Status::OK(); } - virtual arrow::Status close() { + virtual arrow::Status close() + { ARROW_RETURN_NOT_OK(close_read_table_writer()); ARROW_RETURN_NOT_OK(close_signal_table_writer()); return arrow::Status::OK(); } - bool is_closed() const { + bool is_closed() const + { assert(!!m_read_table_writer == !!m_signal_table_writer); return !m_signal_table_writer; } - arrow::MemoryPool* pool() const { return m_pool; } + arrow::MemoryPool * pool() const { return m_pool; } private: DictionaryWriters m_read_table_dict_writers; @@ -172,39 +194,44 @@ class FileWriterImpl { boost::optional m_read_table_writer; boost::optional m_signal_table_writer; std::uint32_t m_signal_chunk_size; - arrow::MemoryPool* m_pool; + arrow::MemoryPool * m_pool; }; class CombinedFileWriterImpl : public FileWriterImpl { public: - CombinedFileWriterImpl(std::string const& path, - std::string const& run_info_tmp_path, - std::string const& reads_tmp_path, - std::int64_t signal_file_start_offset, - boost::uuids::uuid const& section_marker, - boost::uuids::uuid const& file_identifier, - std::string const& software_name, - DictionaryWriters&& dict_writers, - RunInfoTableWriter&& run_info_table_writer, - ReadTableWriter&& read_table_writer, - SignalTableWriter&& signal_table_writer, - std::uint32_t signal_chunk_size, - arrow::MemoryPool* pool) - : FileWriterImpl(std::move(dict_writers), - std::move(run_info_table_writer), - std::move(read_table_writer), - std::move(signal_table_writer), - signal_chunk_size, - pool), - m_path(path), - m_run_info_tmp_path(run_info_tmp_path), - m_reads_tmp_path(reads_tmp_path), - m_signal_file_start_offset(signal_file_start_offset), - m_section_marker(section_marker), - m_file_identifier(file_identifier), - m_software_name(software_name) {} - - arrow::Status close() override { + CombinedFileWriterImpl( + std::string const & path, + std::string const & run_info_tmp_path, + std::string const & reads_tmp_path, + std::int64_t signal_file_start_offset, + boost::uuids::uuid const & section_marker, + boost::uuids::uuid const & file_identifier, + std::string const & software_name, + DictionaryWriters && dict_writers, + RunInfoTableWriter && run_info_table_writer, + ReadTableWriter && read_table_writer, + SignalTableWriter && signal_table_writer, + std::uint32_t signal_chunk_size, + arrow::MemoryPool * pool) + : FileWriterImpl( + std::move(dict_writers), + std::move(run_info_table_writer), + std::move(read_table_writer), + std::move(signal_table_writer), + signal_chunk_size, + pool) + , m_path(path) + , m_run_info_tmp_path(run_info_tmp_path) + , m_reads_tmp_path(reads_tmp_path) + , m_signal_file_start_offset(signal_file_start_offset) + , m_section_marker(section_marker) + , m_file_identifier(file_identifier) + , m_software_name(software_name) + { + } + + arrow::Status close() override + { if (is_closed()) { return arrow::Status::OK(); } @@ -226,33 +253,44 @@ class CombinedFileWriterImpl : public FileWriterImpl { ARROW_RETURN_NOT_OK(combined_file_utils::write_section_marker(file, m_section_marker)); auto file_location_for_full_file = - [&](std::string const& filename) -> arrow::Result { + [&](std::string const & filename) -> arrow::Result { ARROW_ASSIGN_OR_RAISE(auto file, arrow::io::ReadableFile::Open(filename, pool())); ARROW_ASSIGN_OR_RAISE(auto size, file->GetSize()); return FileLocation{filename, 0, std::size_t(size)}; }; // Write in run_info table: - ARROW_ASSIGN_OR_RAISE(auto run_info_location, - file_location_for_full_file(m_run_info_tmp_path)); - ARROW_ASSIGN_OR_RAISE(auto run_info_info_table, - combined_file_utils::write_file_and_marker( - pool(), file, run_info_location, - combined_file_utils::SubFileCleanup::CleanupOriginalFile, - m_section_marker)); + ARROW_ASSIGN_OR_RAISE( + auto run_info_location, file_location_for_full_file(m_run_info_tmp_path)); + ARROW_ASSIGN_OR_RAISE( + auto run_info_info_table, + combined_file_utils::write_file_and_marker( + pool(), + file, + run_info_location, + combined_file_utils::SubFileCleanup::CleanupOriginalFile, + m_section_marker)); // Write in read table: ARROW_ASSIGN_OR_RAISE(auto reads_location, file_location_for_full_file(m_reads_tmp_path)); - ARROW_ASSIGN_OR_RAISE(auto reads_info_table, - combined_file_utils::write_file_and_marker( - pool(), file, reads_location, - combined_file_utils::SubFileCleanup::CleanupOriginalFile, - m_section_marker)); + ARROW_ASSIGN_OR_RAISE( + auto reads_info_table, + combined_file_utils::write_file_and_marker( + pool(), + file, + reads_location, + combined_file_utils::SubFileCleanup::CleanupOriginalFile, + m_section_marker)); // Write full file footer: ARROW_RETURN_NOT_OK(combined_file_utils::write_footer( - file, m_section_marker, m_file_identifier, m_software_name, signal_table, - run_info_info_table, reads_info_table)); + file, + m_section_marker, + m_file_identifier, + m_software_name, + signal_table, + run_info_info_table, + reads_info_table)); return arrow::Status::OK(); } @@ -266,52 +304,61 @@ class CombinedFileWriterImpl : public FileWriterImpl { std::string m_software_name; }; -FileWriter::FileWriter(std::unique_ptr&& impl) : m_impl(std::move(impl)) {} +FileWriter::FileWriter(std::unique_ptr && impl) : m_impl(std::move(impl)) {} FileWriter::~FileWriter() { (void)close(); } arrow::Status FileWriter::close() { return m_impl->close(); } -arrow::Status FileWriter::add_complete_read(ReadData const& read_data, - gsl::span const& signal) { +arrow::Status FileWriter::add_complete_read( + ReadData const & read_data, + gsl::span const & signal) +{ return m_impl->add_complete_read(read_data, signal); } -arrow::Status FileWriter::add_complete_read(ReadData const& read_data, - gsl::span const& signal_rows, - std::uint64_t signal_duration) { +arrow::Status FileWriter::add_complete_read( + ReadData const & read_data, + gsl::span const & signal_rows, + std::uint64_t signal_duration) +{ return m_impl->add_complete_read(read_data, signal_rows, signal_duration); } pod5::Result> FileWriter::add_signal( - boost::uuids::uuid const& read_id, - gsl::span const& signal) { + boost::uuids::uuid const & read_id, + gsl::span const & signal) +{ return m_impl->add_signal(read_id, signal); } pod5::Result FileWriter::add_pre_compressed_signal( - boost::uuids::uuid const& read_id, - gsl::span const& signal_bytes, - std::uint32_t sample_count) { + boost::uuids::uuid const & read_id, + gsl::span const & signal_bytes, + std::uint32_t sample_count) +{ return m_impl->add_pre_compressed_signal(read_id, signal_bytes, sample_count); } -pod5::Result FileWriter::lookup_end_reason( - ReadEndReason end_reason) const { +pod5::Result FileWriter::lookup_end_reason(ReadEndReason end_reason) const +{ return m_impl->lookup_end_reason(end_reason); } -pod5::Result FileWriter::add_pore_type(std::string const& pore_type_data) { +pod5::Result FileWriter::add_pore_type(std::string const & pore_type_data) +{ return m_impl->add_pore_type(pore_type_data); } -pod5::Result FileWriter::add_run_info(RunInfoData const& run_info_data) { +pod5::Result FileWriter::add_run_info(RunInfoData const & run_info_data) +{ return m_impl->add_run_info(run_info_data); } SignalType FileWriter::signal_type() const { return m_impl->signal_type(); } -pod5::Result make_dictionary_writers(arrow::MemoryPool* pool) { +pod5::Result make_dictionary_writers(arrow::MemoryPool * pool) +{ FileWriterImpl::DictionaryWriters writers; ARROW_ASSIGN_OR_RAISE(writers.end_reason_writer, pod5::make_end_reason_writer(pool)); @@ -322,9 +369,10 @@ pod5::Result make_dictionary_writers(arrow::M } pod5::Result> create_file_writer( - std::string const& path, - std::string const& writing_software_name, - FileWriterOptions const& options) { + std::string const & path, + std::string const & writing_software_name, + FileWriterOptions const & options) +{ auto pool = options.memory_pool(); if (!pool) { return Status::Invalid("Invalid memory pool specified for file writer"); @@ -345,34 +393,41 @@ pod5::Result> create_file_writer( auto const file_identifier = uuid_gen(); ARROW_ASSIGN_OR_RAISE(auto current_version, parse_version_number(Pod5Version)); - ARROW_ASSIGN_OR_RAISE(auto file_schema_metadata, - make_schema_key_value_metadata( - {file_identifier, writing_software_name, current_version})); + ARROW_ASSIGN_OR_RAISE( + auto file_schema_metadata, + make_schema_key_value_metadata({file_identifier, writing_software_name, current_version})); - auto reads_tmp_path = arrow_path.Parent().ToString() + "/" + - ("." + boost::uuids::to_string(file_identifier) + ".tmp-reads"); - auto run_info_tmp_path = arrow_path.Parent().ToString() + "/" + - ("." + boost::uuids::to_string(file_identifier) + ".tmp-run-info"); + auto reads_tmp_path = arrow_path.Parent().ToString() + "/" + + ("." + boost::uuids::to_string(file_identifier) + ".tmp-reads"); + auto run_info_tmp_path = arrow_path.Parent().ToString() + "/" + + ("." + boost::uuids::to_string(file_identifier) + ".tmp-run-info"); // Prepare the temporary reads file: - ARROW_ASSIGN_OR_RAISE(auto read_table_file, - arrow::io::FileOutputStream::Open(reads_tmp_path, false)); + ARROW_ASSIGN_OR_RAISE( + auto read_table_file, arrow::io::FileOutputStream::Open(reads_tmp_path, false)); auto read_table_file_async = std::make_shared(read_table_file); ARROW_ASSIGN_OR_RAISE( - auto read_table_tmp_writer, - make_read_table_writer(read_table_file_async, file_schema_metadata, - options.read_table_batch_size(), dict_writers.pore_writer, - dict_writers.end_reason_writer, dict_writers.run_info_writer, - pool)); + auto read_table_tmp_writer, + make_read_table_writer( + read_table_file_async, + file_schema_metadata, + options.read_table_batch_size(), + dict_writers.pore_writer, + dict_writers.end_reason_writer, + dict_writers.run_info_writer, + pool)); // Prepare the temporary run_info file: - ARROW_ASSIGN_OR_RAISE(auto run_info_table_file, - arrow::io::FileOutputStream::Open(run_info_tmp_path, false)); + ARROW_ASSIGN_OR_RAISE( + auto run_info_table_file, arrow::io::FileOutputStream::Open(run_info_tmp_path, false)); auto run_info_table_file_async = std::make_shared(run_info_table_file); ARROW_ASSIGN_OR_RAISE( - auto run_info_table_tmp_writer, - make_run_info_table_writer(run_info_table_file_async, file_schema_metadata, - options.run_info_table_batch_size(), pool)); + auto run_info_table_tmp_writer, + make_run_info_table_writer( + run_info_table_file_async, + file_schema_metadata, + options.run_info_table_batch_size(), + pool)); // Prepare the main file - and set up the signal table to write here: ARROW_ASSIGN_OR_RAISE(auto main_file, arrow::io::FileOutputStream::Open(path, false)); @@ -383,17 +438,30 @@ pod5::Result> create_file_writer( // Then place the signal file directly after that: ARROW_ASSIGN_OR_RAISE(auto const signal_table_start, main_file->Tell()); auto signal_file = std::make_shared(main_file); - ARROW_ASSIGN_OR_RAISE(auto signal_table_writer, - make_signal_table_writer(signal_file, file_schema_metadata, - options.signal_table_batch_size(), - options.signal_type(), pool)); + ARROW_ASSIGN_OR_RAISE( + auto signal_table_writer, + make_signal_table_writer( + signal_file, + file_schema_metadata, + options.signal_table_batch_size(), + options.signal_type(), + pool)); // Throw it all together into a writer object: return std::make_unique(std::make_unique( - path, run_info_tmp_path, reads_tmp_path, signal_table_start, section_marker, - file_identifier, writing_software_name, std::move(dict_writers), - std::move(run_info_table_tmp_writer), std::move(read_table_tmp_writer), - std::move(signal_table_writer), options.max_signal_chunk_size(), pool)); + path, + run_info_tmp_path, + reads_tmp_path, + signal_table_start, + section_marker, + file_identifier, + writing_software_name, + std::move(dict_writers), + std::move(run_info_table_tmp_writer), + std::move(read_table_tmp_writer), + std::move(signal_table_writer), + options.max_signal_chunk_size(), + pool)); } } // namespace pod5 diff --git a/c++/pod5_format/file_writer.h b/c++/pod5_format/file_writer.h index c99af0b..a588401 100644 --- a/c++/pod5_format/file_writer.h +++ b/c++/pod5_format/file_writer.h @@ -25,33 +25,42 @@ class POD5_FORMAT_EXPORT FileWriterOptions { FileWriterOptions(); - void set_max_signal_chunk_size(std::uint32_t chunk_size) { + void set_max_signal_chunk_size(std::uint32_t chunk_size) + { m_max_signal_chunk_size = chunk_size; } + std::uint32_t max_signal_chunk_size() const { return m_max_signal_chunk_size; } - void memory_pool(arrow::MemoryPool* memory_pool) { m_memory_pool = memory_pool; } - arrow::MemoryPool* memory_pool() const { return m_memory_pool; } + void memory_pool(arrow::MemoryPool * memory_pool) { m_memory_pool = memory_pool; } + + arrow::MemoryPool * memory_pool() const { return m_memory_pool; } void set_signal_type(SignalType signal_type) { m_signal_type = signal_type; } + SignalType signal_type() const { return m_signal_type; } - void set_signal_table_batch_size(std::size_t batch_size) { + void set_signal_table_batch_size(std::size_t batch_size) + { m_signal_table_batch_size = batch_size; } + std::size_t signal_table_batch_size() const { return m_signal_table_batch_size; } void set_read_table_batch_size(std::size_t batch_size) { m_read_table_batch_size = batch_size; } + std::size_t read_table_batch_size() const { return m_read_table_batch_size; } - void set_run_info_table_batch_size(std::size_t batch_size) { + void set_run_info_table_batch_size(std::size_t batch_size) + { m_run_info_table_batch_size = batch_size; } + std::size_t run_info_table_batch_size() const { return m_run_info_table_batch_size; } private: std::uint32_t m_max_signal_chunk_size; - arrow::MemoryPool* m_memory_pool; + arrow::MemoryPool * m_memory_pool; SignalType m_signal_type; std::size_t m_signal_table_batch_size; std::size_t m_read_table_batch_size; @@ -59,34 +68,37 @@ class POD5_FORMAT_EXPORT FileWriterOptions { }; class FileWriterImpl; + class POD5_FORMAT_EXPORT FileWriter { public: - FileWriter(std::unique_ptr&& impl); + FileWriter(std::unique_ptr && impl); ~FileWriter(); pod5::Status close(); - pod5::Status add_complete_read(ReadData const& read_data, - gsl::span const& signal); + pod5::Status add_complete_read( + ReadData const & read_data, + gsl::span const & signal); /// \brief Add a complete with rows already pre appended. - pod5::Status add_complete_read(ReadData const& read_data, - gsl::span const& signal_rows, - std::uint64_t signal_duration); + pod5::Status add_complete_read( + ReadData const & read_data, + gsl::span const & signal_rows, + std::uint64_t signal_duration); pod5::Result> add_signal( - boost::uuids::uuid const& read_id, - gsl::span const& signal); + boost::uuids::uuid const & read_id, + gsl::span const & signal); pod5::Result add_pre_compressed_signal( - boost::uuids::uuid const& read_id, - gsl::span const& signal_bytes, - std::uint32_t sample_count); + boost::uuids::uuid const & read_id, + gsl::span const & signal_bytes, + std::uint32_t sample_count); // Find or create an end reason index representing this read end reason. pod5::Result lookup_end_reason(ReadEndReason end_reason) const; - pod5::Result add_pore_type(std::string const& pore_type_data); - pod5::Result add_run_info(RunInfoData const& run_info_data); + pod5::Result add_pore_type(std::string const & pore_type_data); + pod5::Result add_run_info(RunInfoData const & run_info_data); SignalType signal_type() const; @@ -95,8 +107,8 @@ class POD5_FORMAT_EXPORT FileWriter { }; POD5_FORMAT_EXPORT pod5::Result> create_file_writer( - std::string const& path, - std::string const& writing_software_name, - FileWriterOptions const& options = {}); + std::string const & path, + std::string const & writing_software_name, + FileWriterOptions const & options = {}); } // namespace pod5 diff --git a/c++/pod5_format/internal/async_output_stream.h b/c++/pod5_format/internal/async_output_stream.h index 62a6d1c..5230e79 100644 --- a/c++/pod5_format/internal/async_output_stream.h +++ b/c++/pod5_format/internal/async_output_stream.h @@ -13,29 +13,34 @@ class AsyncOutputStream : public arrow::io::OutputStream { public: - AsyncOutputStream(std::shared_ptr const& main_stream) - : m_has_error(false), - m_submitted_writes(0), - m_completed_writes(0), - m_submitted_byte_writes(0), - m_completed_byte_writes(0), - m_exit(false), - m_main_stream(main_stream), - m_write_thread([&] { run_write_thread(); }) {} - - ~AsyncOutputStream() { + AsyncOutputStream(std::shared_ptr const & main_stream) + : m_has_error(false) + , m_submitted_writes(0) + , m_completed_writes(0) + , m_submitted_byte_writes(0) + , m_completed_byte_writes(0) + , m_exit(false) + , m_main_stream(main_stream) + , m_write_thread([&] { run_write_thread(); }) + { + } + + ~AsyncOutputStream() + { (void)Flush(); m_exit = true; m_write_thread.join(); } - virtual arrow::Status Close() override { + virtual arrow::Status Close() override + { ARROW_RETURN_NOT_OK(Flush()); m_exit = true; return m_main_stream->Close(); } - arrow::Future<> CloseAsync() override { + arrow::Future<> CloseAsync() override + { ARROW_RETURN_NOT_OK(Flush()); m_exit = true; return m_main_stream->CloseAsync(); @@ -47,14 +52,16 @@ class AsyncOutputStream : public arrow::io::OutputStream { bool closed() const override { return m_main_stream->closed(); } - arrow::Status Write(const void* data, int64_t nbytes) override { + arrow::Status Write(void const * data, int64_t nbytes) override + { ARROW_ASSIGN_OR_RAISE(std::shared_ptr buffer, arrow::AllocateBuffer(nbytes)); - auto const char_data = static_cast(data); + auto const char_data = static_cast(data); std::copy(char_data, char_data + nbytes, buffer->mutable_data()); return Write(buffer); } - arrow::Status Write(const std::shared_ptr& data) override { + arrow::Status Write(std::shared_ptr const & data) override + { if (m_has_error) { return *m_error; } @@ -75,7 +82,8 @@ class AsyncOutputStream : public arrow::io::OutputStream { return arrow::Status::OK(); } - arrow::Status Flush() override { + arrow::Status Flush() override + { if (m_has_error) { return *m_error; } @@ -89,13 +97,15 @@ class AsyncOutputStream : public arrow::io::OutputStream { } private: - void run_write_thread() { + void run_write_thread() + { while (!m_exit) { std::shared_ptr buffer; { std::unique_lock lock(m_write_mutex); - m_work_available.wait_for(lock, std::chrono::milliseconds(100), - [&] { return !m_write_requests.empty() || m_exit; }); + m_work_available.wait_for(lock, std::chrono::milliseconds(100), [&] { + return !m_write_requests.empty() || m_exit; + }); if (!m_write_requests.size()) { continue; } diff --git a/c++/pod5_format/internal/combined_file_utils.h b/c++/pod5_format/internal/combined_file_utils.h index bf43169..60bd2a1 100644 --- a/c++/pod5_format/internal/combined_file_utils.h +++ b/c++/pod5_format/internal/combined_file_utils.h @@ -14,15 +14,15 @@ #include -namespace pod5 { +namespace pod5 { namespace combined_file_utils { -namespace combined_file_utils { +static constexpr std::array + FILE_SIGNATURE{'\213', 'P', 'O', 'D', '\r', '\n', '\032', '\n'}; -static constexpr std::array FILE_SIGNATURE{'\213', 'P', 'O', 'D', - '\r', '\n', '\032', '\n'}; - -inline pod5::Status pad_file(std::shared_ptr const& sink, - std::uint32_t pad_to_size) { +inline pod5::Status pad_file( + std::shared_ptr const & sink, + std::uint32_t pad_to_size) +{ ARROW_ASSIGN_OR_RAISE(auto const current_byte_location, sink->Tell()); auto const bytes_to_write = pad_to_size - (current_byte_location % pad_to_size); if (bytes_to_write == pad_to_size) { @@ -33,22 +33,29 @@ inline pod5::Status pad_file(std::shared_ptr const& sin return sink->Write(zeroes.data(), bytes_to_write); } -inline pod5::Status write_file_signature(std::shared_ptr const& sink) { +inline pod5::Status write_file_signature(std::shared_ptr const & sink) +{ return sink->Write(FILE_SIGNATURE.data(), FILE_SIGNATURE.size()); } -inline pod5::Status write_section_marker(std::shared_ptr const& sink, - boost::uuids::uuid const& section_marker) { - return sink->Write(section_marker.begin(), - std::distance(section_marker.begin(), section_marker.end())); + +inline pod5::Status write_section_marker( + std::shared_ptr const & sink, + boost::uuids::uuid const & section_marker) +{ + return sink->Write( + section_marker.begin(), std::distance(section_marker.begin(), section_marker.end())); } -inline pod5::Status write_combined_header(std::shared_ptr const& sink, - boost::uuids::uuid const& section_marker) { +inline pod5::Status write_combined_header( + std::shared_ptr const & sink, + boost::uuids::uuid const & section_marker) +{ ARROW_RETURN_NOT_OK(write_file_signature(sink)); return write_section_marker(sink, section_marker); } -inline pod5::Status write_footer_magic(std::shared_ptr const& sink) { +inline pod5::Status write_footer_magic(std::shared_ptr const & sink) +{ return sink->Write("FOOTER\0\0", 8); } @@ -61,10 +68,11 @@ struct ParsedFileInfo : FileInfo { std::string file_path; std::shared_ptr file; - arrow::Status from_full_file(std::string in_file_path) { + arrow::Status from_full_file(std::string in_file_path) + { file_path = in_file_path; ARROW_ASSIGN_OR_RAISE( - file, arrow::io::MemoryMappedFile::Open(in_file_path, arrow::io::FileMode::READ)); + file, arrow::io::MemoryMappedFile::Open(in_file_path, arrow::io::FileMode::READ)); file_start_offset = 0; ARROW_ASSIGN_OR_RAISE(file_length, file->GetSize()); return arrow::Status::OK(); @@ -72,48 +80,64 @@ struct ParsedFileInfo : FileInfo { }; inline pod5::Result write_footer_flatbuffer( - std::shared_ptr const& sink, - boost::uuids::uuid const& file_identifier, - std::string const& software_name, - FileInfo const& signal_table, - FileInfo const& run_info_table, - FileInfo const& reads_table) { + std::shared_ptr const & sink, + boost::uuids::uuid const & file_identifier, + std::string const & software_name, + FileInfo const & signal_table, + FileInfo const & run_info_table, + FileInfo const & reads_table) +{ flatbuffers::FlatBufferBuilder builder(1024); auto signal_file = Minknow::ReadsFormat::CreateEmbeddedFile( - builder, signal_table.file_start_offset, signal_table.file_length, - Minknow::ReadsFormat::Format_FeatherV2, Minknow::ReadsFormat::ContentType_SignalTable); + builder, + signal_table.file_start_offset, + signal_table.file_length, + Minknow::ReadsFormat::Format_FeatherV2, + Minknow::ReadsFormat::ContentType_SignalTable); auto run_info_file = Minknow::ReadsFormat::CreateEmbeddedFile( - builder, run_info_table.file_start_offset, run_info_table.file_length, - Minknow::ReadsFormat::Format_FeatherV2, Minknow::ReadsFormat::ContentType_RunInfoTable); + builder, + run_info_table.file_start_offset, + run_info_table.file_length, + Minknow::ReadsFormat::Format_FeatherV2, + Minknow::ReadsFormat::ContentType_RunInfoTable); auto reads_file = Minknow::ReadsFormat::CreateEmbeddedFile( - builder, reads_table.file_start_offset, reads_table.file_length, - Minknow::ReadsFormat::Format_FeatherV2, Minknow::ReadsFormat::ContentType_ReadsTable); + builder, + reads_table.file_start_offset, + reads_table.file_length, + Minknow::ReadsFormat::Format_FeatherV2, + Minknow::ReadsFormat::ContentType_ReadsTable); const std::vector> files{ - signal_file, run_info_file, reads_file}; + signal_file, run_info_file, reads_file}; auto footer = Minknow::ReadsFormat::CreateFooterDirect( - builder, boost::uuids::to_string(file_identifier).c_str(), software_name.c_str(), - Pod5Version.c_str(), &files); + builder, + boost::uuids::to_string(file_identifier).c_str(), + software_name.c_str(), + Pod5Version.c_str(), + &files); builder.Finish(footer); ARROW_RETURN_NOT_OK(sink->Write(builder.GetBufferPointer(), builder.GetSize())); return builder.GetSize(); } -inline pod5::Status write_footer(std::shared_ptr const& sink, - boost::uuids::uuid const& section_marker, - boost::uuids::uuid const& file_identifier, - std::string const& software_name, - FileInfo const& signal_table, - FileInfo const& run_info_table, - FileInfo const& reads_table) { +inline pod5::Status write_footer( + std::shared_ptr const & sink, + boost::uuids::uuid const & section_marker, + boost::uuids::uuid const & file_identifier, + std::string const & software_name, + FileInfo const & signal_table, + FileInfo const & run_info_table, + FileInfo const & reads_table) +{ ARROW_RETURN_NOT_OK(write_footer_magic(sink)); - ARROW_ASSIGN_OR_RAISE(std::int64_t length, - write_footer_flatbuffer(sink, file_identifier, software_name, - signal_table, run_info_table, reads_table)); + ARROW_ASSIGN_OR_RAISE( + std::int64_t length, + write_footer_flatbuffer( + sink, file_identifier, software_name, signal_table, run_info_table, reads_table)); ARROW_RETURN_NOT_OK(pad_file(sink, 8)); std::int64_t paded_flatbuffer_size = arrow::bit_util::ToLittleEndian(length); @@ -133,11 +157,14 @@ struct ParsedFooter { ParsedFileInfo signal_table; }; -inline pod5::Status check_signature(std::shared_ptr const& file, - std::int64_t offset_in_file) { +inline pod5::Status check_signature( + std::shared_ptr const & file, + std::int64_t offset_in_file) +{ std::array read_signature; - ARROW_ASSIGN_OR_RAISE(auto read_bytes, file->ReadAt(offset_in_file, read_signature.size(), - read_signature.data())); + ARROW_ASSIGN_OR_RAISE( + auto read_bytes, + file->ReadAt(offset_in_file, read_signature.size(), read_signature.data())); if (read_bytes != (std::int16_t)read_signature.size() || read_signature != FILE_SIGNATURE) { return arrow::Status::IOError("Invalid signature in file"); } @@ -145,8 +172,9 @@ inline pod5::Status check_signature(std::shared_ptr return arrow::Status::OK(); } -inline pod5::Result read_footer_flatbuffer( - std::vector const& footer_data) { +inline pod5::Result read_footer_flatbuffer( + std::vector const & footer_data) +{ auto verifier = flatbuffers::Verifier(footer_data.data(), footer_data.size()); if (!verifier.VerifyBuffer()) { return arrow::Status::IOError("Invalid footer found in file"); @@ -155,8 +183,9 @@ inline pod5::Result read_footer_flatbuffer( } inline pod5::Result read_footer( - std::string const& file_path, - std::shared_ptr const& file) { + std::string const & file_path, + std::shared_ptr const & file) +{ // Verify signature at start and end of file: ARROW_RETURN_NOT_OK(check_signature(file, 0)); ARROW_ASSIGN_OR_RAISE(auto const file_size, file->GetSize()); @@ -167,16 +196,18 @@ inline pod5::Result read_footer( footer_length_data_end -= sizeof(boost::uuids::uuid); std::int64_t footer_length = 0; - ARROW_RETURN_NOT_OK(file->ReadAt(footer_length_data_end - sizeof(footer_length), - sizeof(footer_length), &footer_length)); + ARROW_RETURN_NOT_OK(file->ReadAt( + footer_length_data_end - sizeof(footer_length), sizeof(footer_length), &footer_length)); footer_length = arrow::bit_util::FromLittleEndian(footer_length); std::vector footer_data; footer_data.resize(footer_length); ARROW_ASSIGN_OR_RAISE( - auto read_bytes, - file->ReadAt(footer_length_data_end - sizeof(footer_length) - footer_length, - footer_length, footer_data.data())); + auto read_bytes, + file->ReadAt( + footer_length_data_end - sizeof(footer_length) - footer_length, + footer_length, + footer_data.data())); if (read_bytes != footer_length) { return arrow::Status::IOError("Failed to read footer data"); } @@ -188,10 +219,10 @@ inline pod5::Result read_footer( } try { footer.file_identifier = - boost::lexical_cast(fb_footer->file_identifier()->str()); - } catch (boost::bad_lexical_cast const&) { - return Status::IOError("Invalid file_identifier in file: '", - fb_footer->file_identifier()->str(), "'"); + boost::lexical_cast(fb_footer->file_identifier()->str()); + } catch (boost::bad_lexical_cast const &) { + return Status::IOError( + "Invalid file_identifier in file: '", fb_footer->file_identifier()->str(), "'"); } if (!fb_footer->software()) { @@ -241,41 +272,50 @@ inline pod5::Result read_footer( class SubFile : public arrow::io::internal::RandomAccessFileConcurrencyWrapper { public: - SubFile(std::shared_ptr main_file, - std::int64_t sub_file_offset, - std::int64_t sub_file_length) - : m_file(std::move(main_file)), - m_sub_file_offset(sub_file_offset), - m_sub_file_length(sub_file_length) {} + SubFile( + std::shared_ptr main_file, + std::int64_t sub_file_offset, + std::int64_t sub_file_length) + : m_file(std::move(main_file)) + , m_sub_file_offset(sub_file_offset) + , m_sub_file_length(sub_file_length) + { + } protected: arrow::Status DoClose() { return m_file->Close(); } bool closed() const override { return m_file->closed(); } - arrow::Result DoTell() const { + arrow::Result DoTell() const + { ARROW_ASSIGN_OR_RAISE(auto t, m_file->Tell()); return t - m_sub_file_offset; } - arrow::Status DoSeek(int64_t offset) { + arrow::Status DoSeek(int64_t offset) + { offset += m_sub_file_offset; return m_file->Seek(offset); } - arrow::Result DoRead(int64_t length, void* data) { + arrow::Result DoRead(int64_t length, void * data) + { return m_file->Read(length, data); } - arrow::Result> DoRead(int64_t length) { + arrow::Result> DoRead(int64_t length) + { return m_file->Read(length); } - Result DoReadAt(int64_t position, int64_t nbytes, void* out) { + Result DoReadAt(int64_t position, int64_t nbytes, void * out) + { return m_file->ReadAt(position + m_sub_file_offset, nbytes, out); } - Result> DoReadAt(int64_t position, int64_t nbytes) { + Result> DoReadAt(int64_t position, int64_t nbytes) + { return m_file->ReadAt(position + m_sub_file_offset, nbytes); } @@ -289,39 +329,41 @@ class SubFile : public arrow::io::internal::RandomAccessFileConcurrencyWrapper> open_sub_file(ParsedFileInfo file_info) { +inline arrow::Result> open_sub_file(ParsedFileInfo file_info) +{ if (!file_info.file) { return arrow::Status::Invalid("Failed to open file from footer"); } // Restrict our open file to just the run info section: - return std::make_shared(file_info.file, file_info.file_start_offset, - file_info.file_length); + return std::make_shared( + file_info.file, file_info.file_start_offset, file_info.file_length); } enum class SubFileCleanup { CleanupOriginalFile, LeaveOrignalFile }; inline arrow::Result write_file( - arrow::MemoryPool* pool, - std::shared_ptr const& file, - FileLocation const& file_location, - SubFileCleanup cleanup_mode) { + arrow::MemoryPool * pool, + std::shared_ptr const & file, + FileLocation const & file_location, + SubFileCleanup cleanup_mode) +{ combined_file_utils::FileInfo table_data; // Record file start location in bytes within the main file: ARROW_ASSIGN_OR_RAISE(table_data.file_start_offset, file->Tell()); { // Stream out the reads table into the main file: - ARROW_ASSIGN_OR_RAISE(auto reads_table_file_in, - arrow::io::ReadableFile::Open(file_location.file_path, pool)); + ARROW_ASSIGN_OR_RAISE( + auto reads_table_file_in, arrow::io::ReadableFile::Open(file_location.file_path, pool)); ARROW_RETURN_NOT_OK(reads_table_file_in->Seek(file_location.offset)); std::int64_t copied_bytes = 0; std::int64_t target_chunk_size = 10 * 1024 * 1024; // Read in 10MB of data at a time std::vector read_data(target_chunk_size); while (copied_bytes < std::int64_t(file_location.size)) { std::size_t const to_read = - std::min(file_location.size - copied_bytes, target_chunk_size); - ARROW_ASSIGN_OR_RAISE(auto const read_bytes, - reads_table_file_in->Read(to_read, read_data.data())); + std::min(file_location.size - copied_bytes, target_chunk_size); + ARROW_ASSIGN_OR_RAISE( + auto const read_bytes, reads_table_file_in->Read(to_read, read_data.data())); copied_bytes += read_bytes; ARROW_RETURN_NOT_OK(file->Write(read_data.data(), read_bytes)); } @@ -333,8 +375,9 @@ inline arrow::Result write_file( if (cleanup_mode == SubFileCleanup::CleanupOriginalFile) { // Clean up the tmp read path: - ARROW_ASSIGN_OR_RAISE(auto arrow_path, ::arrow::internal::PlatformFilename::FromString( - file_location.file_path)); + ARROW_ASSIGN_OR_RAISE( + auto arrow_path, + ::arrow::internal::PlatformFilename::FromString(file_location.file_path)); ARROW_RETURN_NOT_OK(arrow::internal::DeleteFile(arrow_path)); } @@ -342,11 +385,12 @@ inline arrow::Result write_file( } inline arrow::Result write_file_and_marker( - arrow::MemoryPool* pool, - std::shared_ptr const& file, - FileLocation const& file_location, - SubFileCleanup cleanup_mode, - boost::uuids::uuid const& section_marker) { + arrow::MemoryPool * pool, + std::shared_ptr const & file, + FileLocation const & file_location, + SubFileCleanup cleanup_mode, + boost::uuids::uuid const & section_marker) +{ ARROW_ASSIGN_OR_RAISE(auto file_info, write_file(pool, file, file_location, cleanup_mode)); // Pad file to 8 bytes and mark section: ARROW_RETURN_NOT_OK(combined_file_utils::pad_file(file, 8)); @@ -354,5 +398,4 @@ inline arrow::Result write_file_and_marker( return file_info; } -} // namespace combined_file_utils -} // namespace pod5 +}} // namespace pod5::combined_file_utils diff --git a/c++/pod5_format/migration/migration.cpp b/c++/pod5_format/migration/migration.cpp index 5b17473..0453697 100644 --- a/c++/pod5_format/migration/migration.cpp +++ b/c++/pod5_format/migration/migration.cpp @@ -4,18 +4,19 @@ namespace pod5 { -Result> MakeTmpDir(char const* suffix) { +Result> MakeTmpDir(char const * suffix) +{ std::uint32_t counter = 0; while (counter < 3) { std::string tmp_path = std::string{".tmp_"} + suffix; - std::default_random_engine gen(static_cast( - arrow::internal::GetRandomSeed())); + std::default_random_engine gen( + static_cast(arrow::internal::GetRandomSeed())); std::uniform_int_distribution dist(0, 1024); tmp_path += "_" + std::to_string(dist(gen)); - ARROW_ASSIGN_OR_RAISE(auto filename, - arrow::internal::PlatformFilename::FromString(tmp_path)); + ARROW_ASSIGN_OR_RAISE( + auto filename, arrow::internal::PlatformFilename::FromString(tmp_path)); ARROW_ASSIGN_OR_RAISE(auto created, CreateDir(filename)); if (created) { return std::make_unique(std::move(filename)); diff --git a/c++/pod5_format/migration/migration.h b/c++/pod5_format/migration/migration.h index 1e00e7c..20217bc 100644 --- a/c++/pod5_format/migration/migration.h +++ b/c++/pod5_format/migration/migration.h @@ -10,30 +10,33 @@ namespace pod5 { class TemporaryDir { public: - TemporaryDir(arrow::internal::PlatformFilename&& path) : m_path(path) {} + TemporaryDir(arrow::internal::PlatformFilename && path) : m_path(path) {} ~TemporaryDir() { (void)::arrow::internal::DeleteDirTree(m_path); } - arrow::internal::PlatformFilename const& path() { return m_path; }; + arrow::internal::PlatformFilename const & path() { return m_path; }; private: arrow::internal::PlatformFilename m_path; }; -Result> MakeTmpDir(char const* suffix); +Result> MakeTmpDir(char const * suffix); class MigrationResult { public: - MigrationResult(combined_file_utils::ParsedFooter const& footer) : m_footer(footer) {} - MigrationResult(MigrationResult&&) = default; - MigrationResult& operator=(MigrationResult&&) = default; - MigrationResult(MigrationResult const&) = delete; - MigrationResult& operator=(MigrationResult const&) = delete; + MigrationResult(combined_file_utils::ParsedFooter const & footer) : m_footer(footer) {} - combined_file_utils::ParsedFooter& footer() { return m_footer; } - combined_file_utils::ParsedFooter const& footer() const { return m_footer; } + MigrationResult(MigrationResult &&) = default; + MigrationResult & operator=(MigrationResult &&) = default; + MigrationResult(MigrationResult const &) = delete; + MigrationResult & operator=(MigrationResult const &) = delete; - void add_temp_dir(std::unique_ptr&& temp_dir) { + combined_file_utils::ParsedFooter & footer() { return m_footer; } + + combined_file_utils::ParsedFooter const & footer() const { return m_footer; } + + void add_temp_dir(std::unique_ptr && temp_dir) + { m_temp_dirs.emplace_back(std::move(temp_dir)); } @@ -42,18 +45,22 @@ class MigrationResult { std::vector> m_temp_dirs; }; -arrow::Result migrate_v0_to_v1(MigrationResult&& v0_input, - arrow::MemoryPool* pool); -arrow::Result migrate_v1_to_v2(MigrationResult&& v1_input, - arrow::MemoryPool* pool); -arrow::Result migrate_v2_to_v3(MigrationResult&& v2_input, - arrow::MemoryPool* pool); +arrow::Result migrate_v0_to_v1( + MigrationResult && v0_input, + arrow::MemoryPool * pool); +arrow::Result migrate_v1_to_v2( + MigrationResult && v1_input, + arrow::MemoryPool * pool); +arrow::Result migrate_v2_to_v3( + MigrationResult && v2_input, + arrow::MemoryPool * pool); inline arrow::Result migrate_if_required( - Version writer_version, - const combined_file_utils::ParsedFooter& read_footer, - std::shared_ptr const& source, - arrow::MemoryPool* pool) { + Version writer_version, + combined_file_utils::ParsedFooter const & read_footer, + std::shared_ptr const & source, + arrow::MemoryPool * pool) +{ MigrationResult result{read_footer}; if (writer_version < Version(0, 0, 24)) { diff --git a/c++/pod5_format/migration/migration_utils.h b/c++/pod5_format/migration/migration_utils.h index 5a60709..f03a130 100644 --- a/c++/pod5_format/migration/migration_utils.h +++ b/c++/pod5_format/migration/migration_utils.h @@ -8,9 +8,9 @@ namespace pod5 { template -arrow::Result> make_filled_array(arrow::MemoryPool* pool, - std::size_t row_count, - U default_value) { +arrow::Result> +make_filled_array(arrow::MemoryPool * pool, std::size_t row_count, U default_value) +{ T builder(pool); for (std::size_t i = 0; i < row_count; ++i) { ARROW_RETURN_NOT_OK(builder.Append(default_value)); @@ -19,10 +19,12 @@ arrow::Result> make_filled_array(arrow::MemoryPool return builder.Finish(); } -inline arrow::Status set_column(std::shared_ptr const& schema, - std::vector>& columns, - char const* field_name, - arrow::Result> const& array) { +inline arrow::Status set_column( + std::shared_ptr const & schema, + std::vector> & columns, + char const * field_name, + arrow::Result> const & array) +{ auto field_index = schema->GetFieldIndex(field_name); if (field_index == -1) { return arrow::Status::Invalid("Failed to find field '", field_name, "' during migration."); @@ -37,11 +39,13 @@ inline arrow::Status set_column(std::shared_ptr const& schema, return arrow::Status::OK(); } -inline arrow::Status copy_column(std::shared_ptr const& schema_a, - std::vector>& columns_a, - char const* field_name, - std::shared_ptr const& schema_b, - std::vector>& columns_b) { +inline arrow::Status copy_column( + std::shared_ptr const & schema_a, + std::vector> & columns_a, + char const * field_name, + std::shared_ptr const & schema_b, + std::vector> & columns_b) +{ auto field_index_a = schema_a->GetFieldIndex(field_name); if (field_index_a == -1 || field_index_a >= (std::int64_t)columns_a.size()) { return arrow::Status::Invalid("Failed to find field '", field_name, "' during migration."); @@ -69,23 +73,26 @@ struct Pod5BatchRecordWriter { std::shared_ptr writer; std::shared_ptr schema; - arrow::Status write_batch(std::size_t num_rows, - std::vector> const& columns) { + arrow::Status write_batch( + std::size_t num_rows, + std::vector> const & columns) + { auto const record_batch = arrow::RecordBatch::Make(schema, num_rows, std::move(columns)); return writer->WriteRecordBatch(*record_batch); } }; inline pod5::Result open_record_batch_reader( - arrow::MemoryPool* pool, - combined_file_utils::ParsedFileInfo file_info) { + arrow::MemoryPool * pool, + combined_file_utils::ParsedFileInfo file_info) +{ Pod5BatchRecordReader result; ARROW_ASSIGN_OR_RAISE(auto file, open_sub_file(file_info)); arrow::ipc::IpcReadOptions read_options; read_options.memory_pool = pool; - ARROW_ASSIGN_OR_RAISE(result.reader, - arrow::ipc::RecordBatchFileReader::Open(file, read_options)); + ARROW_ASSIGN_OR_RAISE( + result.reader, arrow::ipc::RecordBatchFileReader::Open(file, read_options)); result.schema = result.reader->schema(); result.metadata = result.schema->metadata(); @@ -97,8 +104,9 @@ inline pod5::Result open_record_batch_reader( } inline pod5::Result> update_metadata( - std::shared_ptr original_metadata, - Version version_to_write) { + std::shared_ptr original_metadata, + Version version_to_write) +{ auto result = original_metadata->Copy(); // Update the reader for the new version: ARROW_RETURN_NOT_OK(result->Set("MINKNOW:pod5_version", version_to_write.to_string())); @@ -106,18 +114,19 @@ inline pod5::Result> update_metad } inline pod5::Result make_record_batch_writer( - arrow::MemoryPool* pool, - std::string path, - std::shared_ptr schema, - std::shared_ptr metadata) { + arrow::MemoryPool * pool, + std::string path, + std::shared_ptr schema, + std::shared_ptr metadata) +{ ARROW_ASSIGN_OR_RAISE(auto file, arrow::io::FileOutputStream::Open(path, false)); arrow::ipc::IpcWriteOptions write_options; write_options.memory_pool = pool; write_options.emit_dictionary_deltas = true; Pod5BatchRecordWriter result; - ARROW_ASSIGN_OR_RAISE(result.writer, - arrow::ipc::MakeFileWriter(file, schema, write_options, metadata)); + ARROW_ASSIGN_OR_RAISE( + result.writer, arrow::ipc::MakeFileWriter(file, schema, write_options, metadata)); result.schema = schema; return result; diff --git a/c++/pod5_format/migration/v0_to_v1.cpp b/c++/pod5_format/migration/v0_to_v1.cpp index bc868fe..c6e6313 100644 --- a/c++/pod5_format/migration/v0_to_v1.cpp +++ b/c++/pod5_format/migration/v0_to_v1.cpp @@ -8,32 +8,35 @@ namespace pod5 { -arrow::Result migrate_v0_to_v1(MigrationResult&& v0_input, - arrow::MemoryPool* pool) { +arrow::Result migrate_v0_to_v1( + MigrationResult && v0_input, + arrow::MemoryPool * pool) +{ ARROW_ASSIGN_OR_RAISE(auto temp_dir, MakeTmpDir("pod5_v0_v1_migration")); ARROW_ASSIGN_OR_RAISE(auto v1_reads_table_path, temp_dir->path().Join("reads_table.arrow")); { - ARROW_ASSIGN_OR_RAISE(auto v0_reader, - open_record_batch_reader(pool, v0_input.footer().reads_table)); + ARROW_ASSIGN_OR_RAISE( + auto v0_reader, open_record_batch_reader(pool, v0_input.footer().reads_table)); - auto v1_new_schama = - arrow::schema({arrow::field("num_minknow_events", arrow::uint64()), - arrow::field("tracked_scaling_scale", arrow::float32()), - arrow::field("tracked_scaling_shift", arrow::float32()), - arrow::field("predicted_scaling_scale", arrow::float32()), - arrow::field("predicted_scaling_shift", arrow::float32()), - arrow::field("num_reads_since_mux_change", arrow::uint32()), - arrow::field("time_since_mux_change", arrow::float32())}); + auto v1_new_schama = arrow::schema( + {arrow::field("num_minknow_events", arrow::uint64()), + arrow::field("tracked_scaling_scale", arrow::float32()), + arrow::field("tracked_scaling_shift", arrow::float32()), + arrow::field("predicted_scaling_scale", arrow::float32()), + arrow::field("predicted_scaling_shift", arrow::float32()), + arrow::field("num_reads_since_mux_change", arrow::uint32()), + arrow::field("time_since_mux_change", arrow::float32())}); - ARROW_ASSIGN_OR_RAISE(auto v1_schema, - arrow::UnifySchemas({v0_reader.schema, v1_new_schama})); + ARROW_ASSIGN_OR_RAISE( + auto v1_schema, arrow::UnifySchemas({v0_reader.schema, v1_new_schama})); - ARROW_ASSIGN_OR_RAISE(auto new_metadata, - update_metadata(v0_reader.metadata, Version(0, 0, 24))); - ARROW_ASSIGN_OR_RAISE(auto v1_writer, - make_record_batch_writer(pool, v1_reads_table_path.ToString(), - v1_schema, new_metadata)); + ARROW_ASSIGN_OR_RAISE( + auto new_metadata, update_metadata(v0_reader.metadata, Version(0, 0, 24))); + ARROW_ASSIGN_OR_RAISE( + auto v1_writer, + make_record_batch_writer( + pool, v1_reads_table_path.ToString(), v1_schema, new_metadata)); for (std::int64_t batch_idx = 0; batch_idx < v0_reader.reader->num_record_batches(); ++batch_idx) { @@ -43,31 +46,45 @@ arrow::Result migrate_v0_to_v1(MigrationResult&& v0_input, // Extend with V1 data: std::vector> columns = v0_batch->columns(); - ARROW_RETURN_NOT_OK( - set_column(v1_schema, columns, "num_minknow_events", - make_filled_array(pool, num_rows, 0))); - ARROW_RETURN_NOT_OK( - set_column(v1_schema, columns, "tracked_scaling_scale", - make_filled_array( - pool, num_rows, std::numeric_limits::quiet_NaN()))); - ARROW_RETURN_NOT_OK( - set_column(v1_schema, columns, "tracked_scaling_shift", - make_filled_array( - pool, num_rows, std::numeric_limits::quiet_NaN()))); - ARROW_RETURN_NOT_OK( - set_column(v1_schema, columns, "predicted_scaling_scale", - make_filled_array( - pool, num_rows, std::numeric_limits::quiet_NaN()))); - ARROW_RETURN_NOT_OK( - set_column(v1_schema, columns, "predicted_scaling_shift", - make_filled_array( - pool, num_rows, std::numeric_limits::quiet_NaN()))); - ARROW_RETURN_NOT_OK( - set_column(v1_schema, columns, "num_reads_since_mux_change", - make_filled_array(pool, num_rows, 0))); - ARROW_RETURN_NOT_OK( - set_column(v1_schema, columns, "time_since_mux_change", - make_filled_array(pool, num_rows, 0.0f))); + ARROW_RETURN_NOT_OK(set_column( + v1_schema, + columns, + "num_minknow_events", + make_filled_array(pool, num_rows, 0))); + ARROW_RETURN_NOT_OK(set_column( + v1_schema, + columns, + "tracked_scaling_scale", + make_filled_array( + pool, num_rows, std::numeric_limits::quiet_NaN()))); + ARROW_RETURN_NOT_OK(set_column( + v1_schema, + columns, + "tracked_scaling_shift", + make_filled_array( + pool, num_rows, std::numeric_limits::quiet_NaN()))); + ARROW_RETURN_NOT_OK(set_column( + v1_schema, + columns, + "predicted_scaling_scale", + make_filled_array( + pool, num_rows, std::numeric_limits::quiet_NaN()))); + ARROW_RETURN_NOT_OK(set_column( + v1_schema, + columns, + "predicted_scaling_shift", + make_filled_array( + pool, num_rows, std::numeric_limits::quiet_NaN()))); + ARROW_RETURN_NOT_OK(set_column( + v1_schema, + columns, + "num_reads_since_mux_change", + make_filled_array(pool, num_rows, 0))); + ARROW_RETURN_NOT_OK(set_column( + v1_schema, + columns, + "time_since_mux_change", + make_filled_array(pool, num_rows, 0.0f))); ARROW_RETURN_NOT_OK(v1_writer.write_batch(num_rows, std::move(columns))); } diff --git a/c++/pod5_format/migration/v1_to_v2.cpp b/c++/pod5_format/migration/v1_to_v2.cpp index 983ef52..3ec6612 100644 --- a/c++/pod5_format/migration/v1_to_v2.cpp +++ b/c++/pod5_format/migration/v1_to_v2.cpp @@ -11,9 +11,10 @@ namespace pod5 { arrow::Result get_num_samples( - std::shared_ptr const& signal_col, - std::size_t row_idx, - std::vector> const& signal_batches) { + std::shared_ptr const & signal_col, + std::size_t row_idx, + std::vector> const & signal_batches) +{ if (signal_batches.empty()) { return 0; } @@ -32,17 +33,17 @@ arrow::Result get_num_samples( auto const batch_row = abs_row - (batch_idx * signal_batch_size); if (batch_idx >= signal_batches.size()) { - return arrow::Status::Invalid("Invalid signal row ", abs_row, - ", cannot find signal batch ", batch_idx); + return arrow::Status::Invalid( + "Invalid signal row ", abs_row, ", cannot find signal batch ", batch_idx); } auto batch = signal_batches[batch_idx]; auto samples_column = - std::static_pointer_cast(batch->GetColumnByName("samples")); + std::static_pointer_cast(batch->GetColumnByName("samples")); if (batch_row >= (std::size_t)samples_column->length()) { - return arrow::Status::Invalid("Invalid signal batch row ", batch_row, ", length is ", - samples_column->length()); + return arrow::Status::Invalid( + "Invalid signal batch row ", batch_row, ", length is ", samples_column->length()); } num_samples += samples_column->Value(batch_row); } @@ -50,32 +51,37 @@ arrow::Result get_num_samples( return num_samples; } -arrow::Result migrate_v1_to_v2(MigrationResult&& v1_input, - arrow::MemoryPool* pool) { +arrow::Result migrate_v1_to_v2( + MigrationResult && v1_input, + arrow::MemoryPool * pool) +{ ARROW_ASSIGN_OR_RAISE(auto temp_dir, MakeTmpDir("pod5_v1_v2_migration")); ARROW_ASSIGN_OR_RAISE(auto v2_reads_table_path, temp_dir->path().Join("reads_table.arrow")); { - ARROW_ASSIGN_OR_RAISE(auto v1_reader, - open_record_batch_reader(pool, v1_input.footer().reads_table)); - ARROW_ASSIGN_OR_RAISE(auto v1_signal_reader, - open_record_batch_reader(pool, v1_input.footer().signal_table)); + ARROW_ASSIGN_OR_RAISE( + auto v1_reader, open_record_batch_reader(pool, v1_input.footer().reads_table)); + ARROW_ASSIGN_OR_RAISE( + auto v1_signal_reader, open_record_batch_reader(pool, v1_input.footer().signal_table)); std::vector> signal_batches( - v1_signal_reader.reader->num_record_batches()); + v1_signal_reader.reader->num_record_batches()); for (std::size_t batch_idx = 0; - batch_idx < (std::size_t)v1_signal_reader.reader->num_record_batches(); ++batch_idx) { - ARROW_ASSIGN_OR_RAISE(signal_batches[batch_idx], - v1_signal_reader.reader->ReadRecordBatch(batch_idx)); + batch_idx < (std::size_t)v1_signal_reader.reader->num_record_batches(); + ++batch_idx) + { + ARROW_ASSIGN_OR_RAISE( + signal_batches[batch_idx], v1_signal_reader.reader->ReadRecordBatch(batch_idx)); } auto v2_new_schama = arrow::schema({arrow::field("num_samples", arrow::uint64())}); - ARROW_ASSIGN_OR_RAISE(auto new_metadata, - update_metadata(v1_reader.metadata, Version(0, 0, 32))); - ARROW_ASSIGN_OR_RAISE(auto v2_schema, - arrow::UnifySchemas({v1_reader.schema, v2_new_schama})); - ARROW_ASSIGN_OR_RAISE(auto v2_writer, - make_record_batch_writer(pool, v2_reads_table_path.ToString(), - v2_schema, new_metadata)); + ARROW_ASSIGN_OR_RAISE( + auto new_metadata, update_metadata(v1_reader.metadata, Version(0, 0, 32))); + ARROW_ASSIGN_OR_RAISE( + auto v2_schema, arrow::UnifySchemas({v1_reader.schema, v2_new_schama})); + ARROW_ASSIGN_OR_RAISE( + auto v2_writer, + make_record_batch_writer( + pool, v2_reads_table_path.ToString(), v2_schema, new_metadata)); for (std::int64_t batch_idx = 0; batch_idx < v1_reader.reader->num_record_batches(); ++batch_idx) { @@ -87,15 +93,15 @@ arrow::Result migrate_v1_to_v2(MigrationResult&& v1_input, std::vector> columns = v1_batch->columns(); auto signal_column = - std::static_pointer_cast(v1_batch->GetColumnByName("signal")); + std::static_pointer_cast(v1_batch->GetColumnByName("signal")); arrow::UInt64Builder num_samples_builder; for (std::int64_t row = 0; row < num_rows; ++row) { - ARROW_ASSIGN_OR_RAISE(auto num_samples, - get_num_samples(signal_column, row, signal_batches)); + ARROW_ASSIGN_OR_RAISE( + auto num_samples, get_num_samples(signal_column, row, signal_batches)); ARROW_RETURN_NOT_OK(num_samples_builder.Append(num_samples)); } ARROW_RETURN_NOT_OK( - set_column(v2_schema, columns, "num_samples", num_samples_builder.Finish())); + set_column(v2_schema, columns, "num_samples", num_samples_builder.Finish())); ARROW_RETURN_NOT_OK(v2_writer.write_batch(num_rows, std::move(columns))); } diff --git a/c++/pod5_format/migration/v2_to_v3.cpp b/c++/pod5_format/migration/v2_to_v3.cpp index 3f2e674..39e9f10 100644 --- a/c++/pod5_format/migration/v2_to_v3.cpp +++ b/c++/pod5_format/migration/v2_to_v3.cpp @@ -21,7 +21,8 @@ struct StringDictBuilder { arrow::Int16Builder indices; arrow::StringBuilder items; - arrow::Result> finish() { + arrow::Result> finish() + { ARROW_ASSIGN_OR_RAISE(auto finished_indices, indices.Finish()); ARROW_ASSIGN_OR_RAISE(auto finished_items, items.Finish()); @@ -38,9 +39,11 @@ struct StringDictBuilder { std::unordered_map lookup; }; -arrow::Result get_dict_struct(std::shared_ptr const& batch, - std::size_t row, - char const* field_name) { +arrow::Result get_dict_struct( + std::shared_ptr const & batch, + std::size_t row, + char const * field_name) +{ auto column = batch->GetColumnByName(field_name); if (!column) { return Status::Invalid("Failed to find column ", field_name); @@ -60,9 +63,9 @@ arrow::Result get_dict_struct(std::shared_ptr con } template -arrow::Status append_struct_row(StructRow const& struct_row, - char const* field_name, - Builder& builder) { +arrow::Status +append_struct_row(StructRow const & struct_row, char const * field_name, Builder & builder) +{ auto field_array = struct_row.data->GetFieldByName(field_name); if (!field_array) { return Status::Invalid("Struct is missing ", field_name, " field"); @@ -79,9 +82,11 @@ arrow::Status append_struct_row(StructRow const& struct_row, return builder.Append(typed_field_array->Value(struct_row.dict_item_index)); } -arrow::Status append_struct_row_to_dict(StructRow const& struct_row, - char const* field_name, - StringDictBuilder& builder) { +arrow::Status append_struct_row_to_dict( + StructRow const & struct_row, + char const * field_name, + StringDictBuilder & builder) +{ auto field_array = struct_row.data->GetFieldByName(field_name); if (!field_array) { return Status::Invalid("Struct is missing ", field_name, " field"); @@ -108,60 +113,64 @@ arrow::Status append_struct_row_to_dict(StructRow const& struct_row, return builder.indices.Append(index); } -arrow::Result migrate_v2_to_v3(MigrationResult&& v2_input, - arrow::MemoryPool* pool) { +arrow::Result migrate_v2_to_v3( + MigrationResult && v2_input, + arrow::MemoryPool * pool) +{ ARROW_ASSIGN_OR_RAISE(auto temp_dir, MakeTmpDir("pod5_v2_v3_migration")); ARROW_ASSIGN_OR_RAISE(auto v3_reads_table_path, temp_dir->path().Join("reads_table.arrow")); - ARROW_ASSIGN_OR_RAISE(auto v3_run_info_table_path, - temp_dir->path().Join("run_info_table.arrow")); + ARROW_ASSIGN_OR_RAISE( + auto v3_run_info_table_path, temp_dir->path().Join("run_info_table.arrow")); { - ARROW_ASSIGN_OR_RAISE(auto v2_reader, - open_record_batch_reader(pool, v2_input.footer().reads_table)); - ARROW_ASSIGN_OR_RAISE(auto new_metadata, - update_metadata(v2_reader.metadata, Version(0, 0, 35))); + ARROW_ASSIGN_OR_RAISE( + auto v2_reader, open_record_batch_reader(pool, v2_input.footer().reads_table)); + ARROW_ASSIGN_OR_RAISE( + auto new_metadata, update_metadata(v2_reader.metadata, Version(0, 0, 35))); { auto v3_reads_schema = arrow::schema( - {arrow::field("read_id", uuid()), - arrow::field("signal", arrow::list(arrow::uint64())), - arrow::field("read_number", arrow::uint32()), - arrow::field("start", arrow::uint64()), - arrow::field("median_before", arrow::float32()), - arrow::field("num_minknow_events", arrow::uint64()), - arrow::field("tracked_scaling_scale", arrow::float32()), - arrow::field("tracked_scaling_shift", arrow::float32()), - arrow::field("predicted_scaling_scale", arrow::float32()), - arrow::field("predicted_scaling_shift", arrow::float32()), - arrow::field("num_reads_since_mux_change", arrow::uint32()), - arrow::field("time_since_mux_change", arrow::float32()), - arrow::field("num_samples", arrow::uint64()), - arrow::field("channel", arrow::uint16()), - arrow::field("well", arrow::uint8()), - arrow::field("pore_type", arrow::dictionary(arrow::int16(), arrow::utf8())), - arrow::field("calibration_offset", arrow::float32()), - arrow::field("calibration_scale", arrow::float32()), - arrow::field("end_reason", arrow::dictionary(arrow::int16(), arrow::utf8())), - arrow::field("end_reason_forced", arrow::boolean()), - arrow::field("run_info", arrow::dictionary(arrow::int16(), arrow::utf8()))}, - new_metadata); - ARROW_ASSIGN_OR_RAISE(auto v3_reads_writer, - make_record_batch_writer(pool, v3_reads_table_path.ToString(), - v3_reads_schema, new_metadata)); - - std::vector const columns_to_copy{"read_id", - "signal", - "read_number", - "start", - "median_before", - "num_minknow_events", - "tracked_scaling_scale", - "tracked_scaling_shift", - "predicted_scaling_scale", - "predicted_scaling_shift", - "num_reads_since_mux_change", - "time_since_mux_change", - "num_samples"}; + {arrow::field("read_id", uuid()), + arrow::field("signal", arrow::list(arrow::uint64())), + arrow::field("read_number", arrow::uint32()), + arrow::field("start", arrow::uint64()), + arrow::field("median_before", arrow::float32()), + arrow::field("num_minknow_events", arrow::uint64()), + arrow::field("tracked_scaling_scale", arrow::float32()), + arrow::field("tracked_scaling_shift", arrow::float32()), + arrow::field("predicted_scaling_scale", arrow::float32()), + arrow::field("predicted_scaling_shift", arrow::float32()), + arrow::field("num_reads_since_mux_change", arrow::uint32()), + arrow::field("time_since_mux_change", arrow::float32()), + arrow::field("num_samples", arrow::uint64()), + arrow::field("channel", arrow::uint16()), + arrow::field("well", arrow::uint8()), + arrow::field("pore_type", arrow::dictionary(arrow::int16(), arrow::utf8())), + arrow::field("calibration_offset", arrow::float32()), + arrow::field("calibration_scale", arrow::float32()), + arrow::field("end_reason", arrow::dictionary(arrow::int16(), arrow::utf8())), + arrow::field("end_reason_forced", arrow::boolean()), + arrow::field("run_info", arrow::dictionary(arrow::int16(), arrow::utf8()))}, + new_metadata); + ARROW_ASSIGN_OR_RAISE( + auto v3_reads_writer, + make_record_batch_writer( + pool, v3_reads_table_path.ToString(), v3_reads_schema, new_metadata)); + + std::vector const columns_to_copy{ + "read_id", + "signal", + "read_number", + "start", + "median_before", + "num_minknow_events", + "tracked_scaling_scale", + "tracked_scaling_shift", + "predicted_scaling_scale", + "predicted_scaling_shift", + "num_reads_since_mux_change", + "time_since_mux_change", + "num_samples"}; // Builders for dict columns StringDictBuilder pore_type; @@ -177,9 +186,13 @@ arrow::Result migrate_v2_to_v3(MigrationResult&& v2_input, // Write V3 data: std::vector> v2_columns = v2_batch->columns(); - for (auto const& col_name : columns_to_copy) { - ARROW_RETURN_NOT_OK(copy_column(v2_reader.schema, v2_columns, col_name.data(), - v3_reads_schema, v3_columns)); + for (auto const & col_name : columns_to_copy) { + ARROW_RETURN_NOT_OK(copy_column( + v2_reader.schema, + v2_columns, + col_name.data(), + v3_reads_schema, + v3_columns)); } arrow::UInt16Builder channel; @@ -188,48 +201,51 @@ arrow::Result migrate_v2_to_v3(MigrationResult&& v2_input, arrow::FloatBuilder calibration_scale; arrow::BooleanBuilder end_reason_forced; for (std::int64_t row = 0; row < num_rows; ++row) { - ARROW_ASSIGN_OR_RAISE(auto calibration_data, - get_dict_struct(v2_batch, row, "calibration")); + ARROW_ASSIGN_OR_RAISE( + auto calibration_data, get_dict_struct(v2_batch, row, "calibration")); ARROW_RETURN_NOT_OK(append_struct_row( - calibration_data, "offset", calibration_offset)); + calibration_data, "offset", calibration_offset)); ARROW_RETURN_NOT_OK(append_struct_row( - calibration_data, "scale", calibration_scale)); + calibration_data, "scale", calibration_scale)); ARROW_ASSIGN_OR_RAISE(auto pore_data, get_dict_struct(v2_batch, row, "pore")); ARROW_RETURN_NOT_OK( - append_struct_row(pore_data, "channel", channel)); + append_struct_row(pore_data, "channel", channel)); ARROW_RETURN_NOT_OK( - append_struct_row(pore_data, "well", well)); + append_struct_row(pore_data, "well", well)); ARROW_RETURN_NOT_OK( - append_struct_row_to_dict(pore_data, "pore_type", pore_type)); + append_struct_row_to_dict(pore_data, "pore_type", pore_type)); - ARROW_ASSIGN_OR_RAISE(auto end_reason_data, - get_dict_struct(v2_batch, row, "end_reason")); + ARROW_ASSIGN_OR_RAISE( + auto end_reason_data, get_dict_struct(v2_batch, row, "end_reason")); ARROW_RETURN_NOT_OK( - append_struct_row_to_dict(end_reason_data, "name", end_reason)); + append_struct_row_to_dict(end_reason_data, "name", end_reason)); ARROW_RETURN_NOT_OK(append_struct_row( - end_reason_data, "forced", end_reason_forced)); + end_reason_data, "forced", end_reason_forced)); - ARROW_ASSIGN_OR_RAISE(auto run_info_data, - get_dict_struct(v2_batch, row, "run_info")); + ARROW_ASSIGN_OR_RAISE( + auto run_info_data, get_dict_struct(v2_batch, row, "run_info")); ARROW_RETURN_NOT_OK( - append_struct_row_to_dict(run_info_data, "acquisition_id", run_info)); + append_struct_row_to_dict(run_info_data, "acquisition_id", run_info)); } - ARROW_RETURN_NOT_OK(set_column(v3_reads_schema, v3_columns, "calibration_offset", - calibration_offset.Finish())); - ARROW_RETURN_NOT_OK(set_column(v3_reads_schema, v3_columns, "calibration_scale", - calibration_scale.Finish())); + ARROW_RETURN_NOT_OK(set_column( + v3_reads_schema, + v3_columns, + "calibration_offset", + calibration_offset.Finish())); + ARROW_RETURN_NOT_OK(set_column( + v3_reads_schema, v3_columns, "calibration_scale", calibration_scale.Finish())); ARROW_RETURN_NOT_OK( - set_column(v3_reads_schema, v3_columns, "channel", channel.Finish())); + set_column(v3_reads_schema, v3_columns, "channel", channel.Finish())); ARROW_RETURN_NOT_OK(set_column(v3_reads_schema, v3_columns, "well", well.Finish())); ARROW_RETURN_NOT_OK( - set_column(v3_reads_schema, v3_columns, "pore_type", pore_type.finish())); + set_column(v3_reads_schema, v3_columns, "pore_type", pore_type.finish())); ARROW_RETURN_NOT_OK( - set_column(v3_reads_schema, v3_columns, "end_reason", end_reason.finish())); - ARROW_RETURN_NOT_OK(set_column(v3_reads_schema, v3_columns, "end_reason_forced", - end_reason_forced.Finish())); + set_column(v3_reads_schema, v3_columns, "end_reason", end_reason.finish())); + ARROW_RETURN_NOT_OK(set_column( + v3_reads_schema, v3_columns, "end_reason_forced", end_reason_forced.Finish())); ARROW_RETURN_NOT_OK( - set_column(v3_reads_schema, v3_columns, "run_info", run_info.finish())); + set_column(v3_reads_schema, v3_columns, "run_info", run_info.finish())); ARROW_RETURN_NOT_OK(v3_reads_writer.write_batch(num_rows, std::move(v3_columns))); } @@ -237,45 +253,46 @@ arrow::Result migrate_v2_to_v3(MigrationResult&& v2_input, } { ARROW_ASSIGN_OR_RAISE( - auto v2_last_batch, - v2_reader.reader->ReadRecordBatch(v2_reader.reader->num_record_batches() - 1)); + auto v2_last_batch, + v2_reader.reader->ReadRecordBatch(v2_reader.reader->num_record_batches() - 1)); auto run_info_column = std::dynamic_pointer_cast( - v2_last_batch->GetColumnByName("run_info")); + v2_last_batch->GetColumnByName("run_info")); if (!run_info_column) { return arrow::Status::Invalid("Failed to find the run info column"); } auto run_info_dict_type = - std::dynamic_pointer_cast(run_info_column->type()); + std::dynamic_pointer_cast(run_info_column->type()); if (!run_info_dict_type) { return arrow::Status::Invalid("Failed to find a run info of the right type"); } auto run_info_items = - std::dynamic_pointer_cast(run_info_column->dictionary()); + std::dynamic_pointer_cast(run_info_column->dictionary()); if (!run_info_items) { return arrow::Status::Invalid("Failed to find a run info items array"); } auto run_info_items_type = - std::dynamic_pointer_cast(run_info_items->type()); + std::dynamic_pointer_cast(run_info_items->type()); if (!run_info_items_type) { return arrow::Status::Invalid( - "Failed to find a run info items array of the right type"); + "Failed to find a run info items array of the right type"); } // Append all the run info dict-struct data to the new table: auto v3_run_info_schema = arrow::schema(run_info_items_type->fields(), new_metadata); - ARROW_ASSIGN_OR_RAISE(auto v3_run_info_writer, - make_record_batch_writer(pool, v3_run_info_table_path.ToString(), - v3_run_info_schema, new_metadata)); + ARROW_ASSIGN_OR_RAISE( + auto v3_run_info_writer, + make_record_batch_writer( + pool, v3_run_info_table_path.ToString(), v3_run_info_schema, new_metadata)); - auto const& fields = run_info_items->fields(); + auto const & fields = run_info_items->fields(); std::vector> v3_columns( - v3_run_info_schema->fields().size()); + v3_run_info_schema->fields().size()); for (std::size_t col = 0; col < v3_columns.size(); ++col) { v3_columns[col] = fields[col]; } - ARROW_RETURN_NOT_OK(v3_run_info_writer.write_batch(run_info_items->length(), - std::move(v3_columns))); + ARROW_RETURN_NOT_OK( + v3_run_info_writer.write_batch(run_info_items->length(), std::move(v3_columns))); ARROW_RETURN_NOT_OK(v3_run_info_writer.writer->Close()); } } @@ -284,7 +301,7 @@ arrow::Result migrate_v2_to_v3(MigrationResult&& v2_input, MigrationResult result = std::move(v2_input); ARROW_RETURN_NOT_OK(result.footer().reads_table.from_full_file(v3_reads_table_path.ToString())); ARROW_RETURN_NOT_OK( - result.footer().run_info_table.from_full_file(v3_run_info_table_path.ToString())); + result.footer().run_info_table.from_full_file(v3_run_info_table_path.ToString())); result.add_temp_dir(std::move(temp_dir)); return result; diff --git a/c++/pod5_format/read_table_reader.cpp b/c++/pod5_format/read_table_reader.cpp index 0d7476e..989bcd2 100644 --- a/c++/pod5_format/read_table_reader.cpp +++ b/c++/pod5_format/read_table_reader.cpp @@ -15,36 +15,44 @@ namespace pod5 { ReadTableRecordBatch::ReadTableRecordBatch( - std::shared_ptr&& batch, - std::shared_ptr const& field_locations) - : TableRecordBatch(std::move(batch)), m_field_locations(field_locations) {} + std::shared_ptr && batch, + std::shared_ptr const & field_locations) +: TableRecordBatch(std::move(batch)) +, m_field_locations(field_locations) +{ +} -ReadTableRecordBatch::ReadTableRecordBatch(ReadTableRecordBatch&& other) - : TableRecordBatch(std::move(other)) { +ReadTableRecordBatch::ReadTableRecordBatch(ReadTableRecordBatch && other) +: TableRecordBatch(std::move(other)) +{ m_field_locations = std::move(other.m_field_locations); } -ReadTableRecordBatch& ReadTableRecordBatch::operator=(ReadTableRecordBatch&& other) { - TableRecordBatch& base = *this; +ReadTableRecordBatch & ReadTableRecordBatch::operator=(ReadTableRecordBatch && other) +{ + TableRecordBatch & base = *this; base = other; m_field_locations = std::move(other.m_field_locations); return *this; } -std::shared_ptr ReadTableRecordBatch::read_id_column() const { +std::shared_ptr ReadTableRecordBatch::read_id_column() const +{ return find_column(batch(), m_field_locations->read_id); } -std::shared_ptr ReadTableRecordBatch::signal_column() const { +std::shared_ptr ReadTableRecordBatch::signal_column() const +{ return find_column(batch(), m_field_locations->signal); } -Result ReadTableRecordBatch::columns() const { +Result ReadTableRecordBatch::columns() const +{ ReadTableRecordColumns result; result.table_version = m_field_locations->table_version(); - auto const& bat = batch(); + auto const & bat = batch(); // V0 fields: result.read_id = find_column(bat, m_field_locations->read_id); @@ -60,11 +68,11 @@ Result ReadTableRecordBatch::columns() const { result.tracked_scaling_scale = find_column(bat, m_field_locations->tracked_scaling_scale); result.tracked_scaling_shift = find_column(bat, m_field_locations->tracked_scaling_shift); result.predicted_scaling_scale = - find_column(bat, m_field_locations->predicted_scaling_scale); + find_column(bat, m_field_locations->predicted_scaling_scale); result.predicted_scaling_shift = - find_column(bat, m_field_locations->predicted_scaling_shift); + find_column(bat, m_field_locations->predicted_scaling_shift); result.num_reads_since_mux_change = - find_column(bat, m_field_locations->num_reads_since_mux_change); + find_column(bat, m_field_locations->num_reads_since_mux_change); result.time_since_mux_change = find_column(bat, m_field_locations->time_since_mux_change); } @@ -88,7 +96,8 @@ Result ReadTableRecordBatch::columns() const { return result; } -Result ReadTableRecordBatch::get_pore_type(std::int16_t pore_index) const { +Result ReadTableRecordBatch::get_pore_type(std::int16_t pore_index) const +{ std::lock_guard l(m_dictionary_access_lock); if (!m_field_locations->pore_type.found_field()) { @@ -98,15 +107,16 @@ Result ReadTableRecordBatch::get_pore_type(std::int16_t pore_index) auto pore_column = find_column(batch(), m_field_locations->pore_type); auto pore_data = std::static_pointer_cast(pore_column->dictionary()); if (pore_index >= pore_data->length()) { - return arrow::Status::IndexError("Invalid index ", pore_index, " for pore array of length ", - pore_data->length()); + return arrow::Status::IndexError( + "Invalid index ", pore_index, " for pore array of length ", pore_data->length()); } return pore_data->Value(pore_index).to_string(); } Result> ReadTableRecordBatch::get_end_reason( - std::int16_t end_reason_index) const { + std::int16_t end_reason_index) const +{ std::lock_guard l(m_dictionary_access_lock); if (!m_field_locations->end_reason.found_field()) { @@ -115,11 +125,13 @@ Result> ReadTableRecordBatch::get_end_reas auto end_reason_column = find_column(batch(), m_field_locations->end_reason); auto end_reason_data = - std::static_pointer_cast(end_reason_column->dictionary()); + std::static_pointer_cast(end_reason_column->dictionary()); if (end_reason_index >= end_reason_data->length()) { - return arrow::Status::IndexError("Invalid index ", end_reason_index, - " for end reason array of length ", - end_reason_data->length()); + return arrow::Status::IndexError( + "Invalid index ", + end_reason_index, + " for end reason array of length ", + end_reason_data->length()); } auto str_value = end_reason_data->Value(end_reason_index).to_string(); @@ -127,7 +139,8 @@ Result> ReadTableRecordBatch::get_end_reas return std::make_pair(end_reason_from_string(str_value), str_value); } -Result ReadTableRecordBatch::get_run_info(std::int16_t run_info_index) const { +Result ReadTableRecordBatch::get_run_info(std::int16_t run_info_index) const +{ std::lock_guard l(m_dictionary_access_lock); if (!m_field_locations->run_info.found_field()) { @@ -136,10 +149,13 @@ Result ReadTableRecordBatch::get_run_info(std::int16_t run_info_ind auto run_info_column = find_column(batch(), m_field_locations->run_info); auto run_info_data = - std::static_pointer_cast(run_info_column->dictionary()); + std::static_pointer_cast(run_info_column->dictionary()); if (run_info_index >= run_info_data->length()) { - return arrow::Status::IndexError("Invalid index ", run_info_index, - " for run info array of length ", run_info_data->length()); + return arrow::Status::IndexError( + "Invalid index ", + run_info_index, + " for run info array of length ", + run_info_data->length()); } return run_info_data->Value(run_info_index).to_string(); @@ -148,27 +164,33 @@ Result ReadTableRecordBatch::get_run_info(std::int16_t run_info_ind //--------------------------------------------------------------------------------------------------------------------- ReadTableReader::ReadTableReader( - std::shared_ptr&& input_source, - std::shared_ptr&& reader, - std::shared_ptr const& field_locations, - SchemaMetadataDescription&& schema_metadata, - arrow::MemoryPool* pool) - : TableReader(std::move(input_source), std::move(reader), std::move(schema_metadata), pool), - m_field_locations(field_locations) {} - -ReadTableReader::ReadTableReader(ReadTableReader&& other) - : TableReader(std::move(other)), - m_field_locations(std::move(other.m_field_locations)), - m_sorted_file_read_ids(std::move(other.m_sorted_file_read_ids)) {} - -ReadTableReader& ReadTableReader::operator=(ReadTableReader&& other) { - static_cast(*this) = std::move(static_cast(*this)); + std::shared_ptr && input_source, + std::shared_ptr && reader, + std::shared_ptr const & field_locations, + SchemaMetadataDescription && schema_metadata, + arrow::MemoryPool * pool) +: TableReader(std::move(input_source), std::move(reader), std::move(schema_metadata), pool) +, m_field_locations(field_locations) +{ +} + +ReadTableReader::ReadTableReader(ReadTableReader && other) +: TableReader(std::move(other)) +, m_field_locations(std::move(other.m_field_locations)) +, m_sorted_file_read_ids(std::move(other.m_sorted_file_read_ids)) +{ +} + +ReadTableReader & ReadTableReader::operator=(ReadTableReader && other) +{ + static_cast(*this) = std::move(static_cast(*this)); m_field_locations = std::move(other.m_field_locations); m_sorted_file_read_ids = std::move(other.m_sorted_file_read_ids); return *this; } -Result ReadTableReader::read_record_batch(std::size_t i) const { +Result ReadTableReader::read_record_batch(std::size_t i) const +{ std::lock_guard l(m_batch_get_mutex); auto record_batch = reader()->ReadRecordBatch(i); if (!record_batch.ok()) { @@ -177,7 +199,8 @@ Result ReadTableReader::read_record_batch(std::size_t i) c return ReadTableRecordBatch{std::move(*record_batch), m_field_locations}; } -Status ReadTableReader::build_read_id_lookup() { +Status ReadTableReader::build_read_id_lookup() +{ if (!m_sorted_file_read_ids.empty()) { return Status::OK(); } @@ -208,8 +231,9 @@ Status ReadTableReader::build_read_id_lookup() { } // Sort by read id for searching later: - std::sort(file_read_ids.begin(), file_read_ids.end(), - [](auto const& a, auto const& b) { return a.id < b.id; }); + std::sort(file_read_ids.begin(), file_read_ids.end(), [](auto const & a, auto const & b) { + return a.id < b.id; + }); // Move data out now we successfully build the index: m_sorted_file_read_ids = std::move(file_read_ids); @@ -217,23 +241,25 @@ Status ReadTableReader::build_read_id_lookup() { return Status::OK(); } -Result ReadTableReader::search_for_read_ids(ReadIdSearchInput const& search_input, - gsl::span const& batch_counts, - gsl::span const& batch_rows) { +Result ReadTableReader::search_for_read_ids( + ReadIdSearchInput const & search_input, + gsl::span const & batch_counts, + gsl::span const & batch_rows) +{ ARROW_RETURN_NOT_OK(build_read_id_lookup()); std::size_t successes = 0; std::vector> batch_data(batch_counts.size()); auto const initial_reserve_size = search_input.read_id_count() / batch_counts.size(); - for (auto& br : batch_data) { + for (auto & br : batch_data) { br.reserve(initial_reserve_size); } auto file_ids_current_it = m_sorted_file_read_ids.begin(); auto const file_ids_end = m_sorted_file_read_ids.end(); for (std::size_t i = 0; i < search_input.read_id_count(); ++i) { - auto const& search_item = search_input[i]; + auto const & search_item = search_input[i]; // Increment file pointer while less than the search term: while (file_ids_current_it->id < search_item.id && file_ids_current_it != file_ids_end) { @@ -249,7 +275,7 @@ Result ReadTableReader::search_for_read_ids(ReadIdSearchInput const std::size_t full_size_so_far = 0; for (std::size_t i = 0; i < batch_data.size(); ++i) { - auto& data = batch_data[i]; + auto & data = batch_data[i]; batch_counts[i] = data.size(); // Ensure the batch indices within the batch are sorted: @@ -267,8 +293,9 @@ Result ReadTableReader::search_for_read_ids(ReadIdSearchInput const //--------------------------------------------------------------------------------------------------------------------- Result make_read_table_reader( - std::shared_ptr const& input, - arrow::MemoryPool* pool) { + std::shared_ptr const & input, + arrow::MemoryPool * pool) +{ arrow::ipc::IpcReadOptions options; options.memory_pool = pool; @@ -278,13 +305,13 @@ Result make_read_table_reader( if (!read_metadata_key_values) { return Status::IOError("Missing metadata on read table schema"); } - ARROW_ASSIGN_OR_RAISE(auto read_metadata, - read_schema_key_value_metadata(read_metadata_key_values)); - ARROW_ASSIGN_OR_RAISE(auto field_locations, - read_read_table_schema(read_metadata, reader->schema())); + ARROW_ASSIGN_OR_RAISE( + auto read_metadata, read_schema_key_value_metadata(read_metadata_key_values)); + ARROW_ASSIGN_OR_RAISE( + auto field_locations, read_read_table_schema(read_metadata, reader->schema())); - return ReadTableReader({input}, std::move(reader), field_locations, std::move(read_metadata), - pool); + return ReadTableReader( + {input}, std::move(reader), field_locations, std::move(read_metadata), pool); } } // namespace pod5 diff --git a/c++/pod5_format/read_table_reader.h b/c++/pod5_format/read_table_reader.h index 1a349e5..78ee24e 100644 --- a/c++/pod5_format/read_table_reader.h +++ b/c++/pod5_format/read_table_reader.h @@ -16,9 +16,11 @@ namespace arrow { class Schema; + namespace io { class RandomAccessFile; } + namespace ipc { class RecordBatchFileReader; } @@ -64,17 +66,18 @@ struct ReadTableRecordColumns { class POD5_FORMAT_EXPORT ReadTableRecordBatch : public TableRecordBatch { public: - ReadTableRecordBatch(std::shared_ptr&& batch, - std::shared_ptr const& field_locations); - ReadTableRecordBatch(ReadTableRecordBatch&&); - ReadTableRecordBatch& operator=(ReadTableRecordBatch&&); + ReadTableRecordBatch( + std::shared_ptr && batch, + std::shared_ptr const & field_locations); + ReadTableRecordBatch(ReadTableRecordBatch &&); + ReadTableRecordBatch & operator=(ReadTableRecordBatch &&); std::shared_ptr read_id_column() const; std::shared_ptr signal_column() const; Result get_pore_type(std::int16_t pore_dict_index) const; Result> get_end_reason( - std::int16_t end_reason_dict_index) const; + std::int16_t end_reason_dict_index) const; Result get_run_info(std::int16_t run_info_dict_index) const; Result columns() const; @@ -86,22 +89,24 @@ class POD5_FORMAT_EXPORT ReadTableRecordBatch : public TableRecordBatch { class POD5_FORMAT_EXPORT ReadTableReader : public TableReader { public: - ReadTableReader(std::shared_ptr&& input_source, - std::shared_ptr&& reader, - std::shared_ptr const& field_locations, - SchemaMetadataDescription&& schema_metadata, - arrow::MemoryPool* pool); + ReadTableReader( + std::shared_ptr && input_source, + std::shared_ptr && reader, + std::shared_ptr const & field_locations, + SchemaMetadataDescription && schema_metadata, + arrow::MemoryPool * pool); - ReadTableReader(ReadTableReader&& other); - ReadTableReader& operator=(ReadTableReader&& other); + ReadTableReader(ReadTableReader && other); + ReadTableReader & operator=(ReadTableReader && other); Result read_record_batch(std::size_t i) const; Status build_read_id_lookup(); - Result search_for_read_ids(ReadIdSearchInput const& search_input, - gsl::span const& batch_counts, - gsl::span const& batch_rows); + Result search_for_read_ids( + ReadIdSearchInput const & search_input, + gsl::span const & batch_counts, + gsl::span const & batch_rows); private: struct IndexData { @@ -117,7 +122,7 @@ class POD5_FORMAT_EXPORT ReadTableReader : public TableReader { }; POD5_FORMAT_EXPORT Result make_read_table_reader( - std::shared_ptr const& sink, - arrow::MemoryPool* pool); + std::shared_ptr const & sink, + arrow::MemoryPool * pool); } // namespace pod5 diff --git a/c++/pod5_format/read_table_schema.cpp b/c++/pod5_format/read_table_schema.cpp index 849f8b0..a4e0320 100644 --- a/c++/pod5_format/read_table_schema.cpp +++ b/c++/pod5_format/read_table_schema.cpp @@ -6,80 +6,71 @@ namespace pod5 { ReadTableSchemaDescription::ReadTableSchemaDescription() - : SchemaDescriptionBase(table_version_from_file_version(current_build_version_number())) - // V0 Fields - , - read_id(this, "read_id", uuid(), ReadTableSpecVersion::v0()), - signal(this, "signal", arrow::list(arrow::uint64()), ReadTableSpecVersion::v0()), - read_number(this, "read_number", arrow::uint32(), ReadTableSpecVersion::v0()), - start(this, "start", arrow::uint64(), ReadTableSpecVersion::v0()), - median_before(this, "median_before", arrow::float32(), ReadTableSpecVersion::v0()), - // V1 Fields - num_minknow_events(this, - "num_minknow_events", - arrow::uint64(), - ReadTableSpecVersion::v1()), - tracked_scaling_scale(this, - "tracked_scaling_scale", - arrow::float32(), - ReadTableSpecVersion::v1()), - tracked_scaling_shift(this, - "tracked_scaling_shift", - arrow::float32(), - ReadTableSpecVersion::v1()), - predicted_scaling_scale(this, - "predicted_scaling_scale", - arrow::float32(), - ReadTableSpecVersion::v1()), - predicted_scaling_shift(this, - "predicted_scaling_shift", - arrow::float32(), - ReadTableSpecVersion::v1()), - num_reads_since_mux_change(this, - "num_reads_since_mux_change", - arrow::uint32(), - ReadTableSpecVersion::v1()), - time_since_mux_change(this, - "time_since_mux_change", - arrow::float32(), - ReadTableSpecVersion::v1()), - // V2 Fields - num_samples(this, "num_samples", arrow::uint64(), ReadTableSpecVersion::v2()), - // V3 Fields - channel(this, "channel", arrow::uint16(), ReadTableSpecVersion::v3()), - well(this, "well", arrow::uint8(), ReadTableSpecVersion::v3()), - pore_type(this, - "pore_type", - arrow::dictionary(arrow::int16(), arrow::utf8()), - ReadTableSpecVersion::v3()), - calibration_offset(this, - "calibration_offset", - arrow::float32(), - ReadTableSpecVersion::v3()), - calibration_scale(this, - "calibration_scale", - arrow::float32(), - ReadTableSpecVersion::v3()), - end_reason(this, - "end_reason", - arrow::dictionary(arrow::int16(), arrow::utf8()), - ReadTableSpecVersion::v3()), - end_reason_forced(this, - "end_reason_forced", - arrow::boolean(), - ReadTableSpecVersion::v3()), - run_info(this, - "run_info", - arrow::dictionary(arrow::int16(), arrow::utf8()), - ReadTableSpecVersion::v3()) {} +: SchemaDescriptionBase(table_version_from_file_version(current_build_version_number())) +// V0 Fields +, read_id(this, "read_id", uuid(), ReadTableSpecVersion::v0()) +, signal(this, "signal", arrow::list(arrow::uint64()), ReadTableSpecVersion::v0()) +, read_number(this, "read_number", arrow::uint32(), ReadTableSpecVersion::v0()) +, start(this, "start", arrow::uint64(), ReadTableSpecVersion::v0()) +, median_before(this, "median_before", arrow::float32(), ReadTableSpecVersion::v0()) +, +// V1 Fields +num_minknow_events(this, "num_minknow_events", arrow::uint64(), ReadTableSpecVersion::v1()) +, tracked_scaling_scale(this, "tracked_scaling_scale", arrow::float32(), ReadTableSpecVersion::v1()) +, tracked_scaling_shift(this, "tracked_scaling_shift", arrow::float32(), ReadTableSpecVersion::v1()) +, predicted_scaling_scale( + this, + "predicted_scaling_scale", + arrow::float32(), + ReadTableSpecVersion::v1()) +, predicted_scaling_shift( + this, + "predicted_scaling_shift", + arrow::float32(), + ReadTableSpecVersion::v1()) +, num_reads_since_mux_change( + this, + "num_reads_since_mux_change", + arrow::uint32(), + ReadTableSpecVersion::v1()) +, time_since_mux_change(this, "time_since_mux_change", arrow::float32(), ReadTableSpecVersion::v1()) +, +// V2 Fields +num_samples(this, "num_samples", arrow::uint64(), ReadTableSpecVersion::v2()) +, +// V3 Fields +channel(this, "channel", arrow::uint16(), ReadTableSpecVersion::v3()) +, well(this, "well", arrow::uint8(), ReadTableSpecVersion::v3()) +, pore_type( + this, + "pore_type", + arrow::dictionary(arrow::int16(), arrow::utf8()), + ReadTableSpecVersion::v3()) +, calibration_offset(this, "calibration_offset", arrow::float32(), ReadTableSpecVersion::v3()) +, calibration_scale(this, "calibration_scale", arrow::float32(), ReadTableSpecVersion::v3()) +, end_reason( + this, + "end_reason", + arrow::dictionary(arrow::int16(), arrow::utf8()), + ReadTableSpecVersion::v3()) +, end_reason_forced(this, "end_reason_forced", arrow::boolean(), ReadTableSpecVersion::v3()) +, run_info( + this, + "run_info", + arrow::dictionary(arrow::int16(), arrow::utf8()), + ReadTableSpecVersion::v3()) +{ +} TableSpecVersion ReadTableSchemaDescription::table_version_from_file_version( - Version file_version) const { + Version file_version) const +{ return ReadTableSpecVersion::latest(); } Result> read_dict_value_struct_type( - std::shared_ptr const& datatype) { + std::shared_ptr const & datatype) +{ if (datatype->id() != arrow::Type::DICTIONARY) { return arrow::Status::Invalid("Dictionary type is not a dictionary"); } @@ -93,15 +84,17 @@ Result> read_dict_value_struct_type( } Result> read_read_table_schema( - SchemaMetadataDescription const& schema_metadata, - std::shared_ptr const& schema) { + SchemaMetadataDescription const & schema_metadata, + std::shared_ptr const & schema) +{ auto result = std::make_shared(); ARROW_RETURN_NOT_OK(ReadTableSchemaDescription::read_schema(result, schema_metadata, schema)); return result; } -Result find_struct_field(std::shared_ptr const& type, char const* name) { +Result find_struct_field(std::shared_ptr const & type, char const * name) +{ int field_idx = type->GetFieldIndex(name); if (field_idx == -1) { return pod5::Status::Invalid("Missing ", name, " field in struct"); diff --git a/c++/pod5_format/read_table_schema.h b/c++/pod5_format/read_table_schema.h index 53ff9a3..25002f2 100644 --- a/c++/pod5_format/read_table_schema.h +++ b/c++/pod5_format/read_table_schema.h @@ -24,15 +24,21 @@ struct SchemaMetadataDescription; class ReadTableSpecVersion { public: static TableSpecVersion v0() { return TableSpecVersion::first_version(); } - static TableSpecVersion v1() { + + static TableSpecVersion v1() + { // Addition of num_minknow_events and scaling parameters return TableSpecVersion::at_version(1); } - static TableSpecVersion v2() { + + static TableSpecVersion v2() + { // Addition of num_samples parameters return TableSpecVersion::at_version(2); } - static TableSpecVersion v3() { + + static TableSpecVersion v3() + { // Flattening of dictionaries into separate table. return TableSpecVersion::at_version(3); } @@ -44,8 +50,8 @@ class ReadTableSchemaDescription : public SchemaDescriptionBase { public: ReadTableSchemaDescription(); - ReadTableSchemaDescription(ReadTableSchemaDescription const&) = delete; - ReadTableSchemaDescription& operator=(ReadTableSchemaDescription const&) = delete; + ReadTableSchemaDescription(ReadTableSchemaDescription const &) = delete; + ReadTableSchemaDescription & operator=(ReadTableSchemaDescription const &) = delete; TableSpecVersion table_version_from_file_version(Version file_version) const override; @@ -81,38 +87,38 @@ class ReadTableSchemaDescription : public SchemaDescriptionBase { // Field Builders only for fields we write in newly generated files. // Should not include fields which are removed in the latest version: using FieldBuilders = FieldBuilder< - // V0 fields - decltype(read_id), - decltype(signal), - decltype(read_number), - decltype(start), - decltype(median_before), - - // V1 fields - decltype(num_minknow_events), - decltype(tracked_scaling_scale), - decltype(tracked_scaling_shift), - decltype(predicted_scaling_scale), - decltype(predicted_scaling_shift), - decltype(num_reads_since_mux_change), - decltype(time_since_mux_change), - - // V2 fields - decltype(num_samples), - - // V3 fields - decltype(channel), - decltype(well), - decltype(pore_type), - decltype(calibration_offset), - decltype(calibration_scale), - decltype(end_reason), - decltype(end_reason_forced), - decltype(run_info)>; + // V0 fields + decltype(read_id), + decltype(signal), + decltype(read_number), + decltype(start), + decltype(median_before), + + // V1 fields + decltype(num_minknow_events), + decltype(tracked_scaling_scale), + decltype(tracked_scaling_shift), + decltype(predicted_scaling_scale), + decltype(predicted_scaling_shift), + decltype(num_reads_since_mux_change), + decltype(time_since_mux_change), + + // V2 fields + decltype(num_samples), + + // V3 fields + decltype(channel), + decltype(well), + decltype(pore_type), + decltype(calibration_offset), + decltype(calibration_scale), + decltype(end_reason), + decltype(end_reason_forced), + decltype(run_info)>; }; POD5_FORMAT_EXPORT Result> read_read_table_schema( - SchemaMetadataDescription const& schema_metadata, - std::shared_ptr const&); + SchemaMetadataDescription const & schema_metadata, + std::shared_ptr const &); } // namespace pod5 diff --git a/c++/pod5_format/read_table_utils.cpp b/c++/pod5_format/read_table_utils.cpp index bffc6ea..942a848 100644 --- a/c++/pod5_format/read_table_utils.cpp +++ b/c++/pod5_format/read_table_utils.cpp @@ -4,8 +4,9 @@ namespace pod5 { -ReadIdSearchInput::ReadIdSearchInput(gsl::span const& input_ids) - : m_search_read_ids(input_ids.size()) { +ReadIdSearchInput::ReadIdSearchInput(gsl::span const & input_ids) +: m_search_read_ids(input_ids.size()) +{ // Copy in search input: for (std::size_t i = 0; i < input_ids.size(); ++i) { m_search_read_ids[i].id = input_ids[i]; @@ -13,8 +14,10 @@ ReadIdSearchInput::ReadIdSearchInput(gsl::span const& } // Sort input based on read id: - std::sort(m_search_read_ids.begin(), m_search_read_ids.end(), - [](auto const& a, auto const& b) { return a.id < b.id; }); + std::sort( + m_search_read_ids.begin(), m_search_read_ids.end(), [](auto const & a, auto const & b) { + return a.id < b.id; + }); } } // namespace pod5 diff --git a/c++/pod5_format/read_table_utils.h b/c++/pod5_format/read_table_utils.h index b376a71..9263add 100644 --- a/c++/pod5_format/read_table_utils.h +++ b/c++/pod5_format/read_table_utils.h @@ -22,6 +22,7 @@ using RunInfoDictionaryIndex = std::int16_t; class ReadData { public: ReadData() = default; + /// \brief Create a new read data structure to add to a read. /// \param read_id The read id for the read entry. /// \param read_number Read number for this read. @@ -31,44 +32,47 @@ class ReadData { /// \param end_reason_forced Boolean value indicating if the read end was forced. /// \param run_info_id The dictionary index of the run info for this read. /// \param num_minknow_events The number of minknow events in the read. - ReadData(boost::uuids::uuid const& read_id, - std::uint32_t read_number, - std::uint64_t start_sample, - std::uint16_t channel, - std::uint8_t well, - PoreDictionaryIndex pore_type, - float calibration_offset, - float calibration_scale, - float median_before, - EndReasonDictionaryIndex end_reason, - bool end_reason_forced, - RunInfoDictionaryIndex run_info, - std::uint64_t num_minknow_events, - float tracked_scaling_scale, - float tracked_scaling_shift, - float predicted_scaling_scale, - float predicted_scaling_shift, - std::uint32_t num_reads_since_mux_change, - float time_since_mux_change) - : read_id(read_id), - read_number(read_number), - start_sample(start_sample), - median_before(median_before), - end_reason(end_reason), - end_reason_forced(end_reason_forced), - run_info(run_info), - num_minknow_events(num_minknow_events), - tracked_scaling_scale(tracked_scaling_scale), - tracked_scaling_shift(tracked_scaling_shift), - predicted_scaling_scale(predicted_scaling_scale), - predicted_scaling_shift(predicted_scaling_shift), - num_reads_since_mux_change(num_reads_since_mux_change), - time_since_mux_change(time_since_mux_change), - channel(channel), - well(well), - pore_type(pore_type), - calibration_offset(calibration_offset), - calibration_scale(calibration_scale) {} + ReadData( + boost::uuids::uuid const & read_id, + std::uint32_t read_number, + std::uint64_t start_sample, + std::uint16_t channel, + std::uint8_t well, + PoreDictionaryIndex pore_type, + float calibration_offset, + float calibration_scale, + float median_before, + EndReasonDictionaryIndex end_reason, + bool end_reason_forced, + RunInfoDictionaryIndex run_info, + std::uint64_t num_minknow_events, + float tracked_scaling_scale, + float tracked_scaling_shift, + float predicted_scaling_scale, + float predicted_scaling_shift, + std::uint32_t num_reads_since_mux_change, + float time_since_mux_change) + : read_id(read_id) + , read_number(read_number) + , start_sample(start_sample) + , median_before(median_before) + , end_reason(end_reason) + , end_reason_forced(end_reason_forced) + , run_info(run_info) + , num_minknow_events(num_minknow_events) + , tracked_scaling_scale(tracked_scaling_scale) + , tracked_scaling_shift(tracked_scaling_shift) + , predicted_scaling_scale(predicted_scaling_scale) + , predicted_scaling_shift(predicted_scaling_shift) + , num_reads_since_mux_change(num_reads_since_mux_change) + , time_since_mux_change(time_since_mux_change) + , channel(channel) + , well(well) + , pore_type(pore_type) + , calibration_offset(calibration_offset) + , calibration_scale(calibration_scale) + { + } // V1 Fields boost::uuids::uuid read_id; @@ -96,72 +100,79 @@ class ReadData { float calibration_scale; }; -inline bool operator==(ReadData const& a, ReadData const& b) { - return a.read_id == b.read_id && a.read_number == b.read_number && - a.start_sample == b.start_sample && a.median_before == b.median_before && - a.end_reason == b.end_reason && a.end_reason_forced == b.end_reason_forced && - a.run_info == b.run_info && a.num_minknow_events == b.num_minknow_events && - a.tracked_scaling_scale == b.tracked_scaling_scale && - a.tracked_scaling_shift == b.tracked_scaling_shift && - a.predicted_scaling_scale == b.predicted_scaling_scale && - a.predicted_scaling_shift == b.predicted_scaling_shift && - a.num_reads_since_mux_change == b.num_reads_since_mux_change && - a.time_since_mux_change == b.time_since_mux_change && a.channel == b.channel && - a.well == b.well && a.pore_type == b.pore_type && - a.calibration_offset == b.calibration_offset && - a.calibration_scale == b.calibration_scale; +inline bool operator==(ReadData const & a, ReadData const & b) +{ + return a.read_id == b.read_id && a.read_number == b.read_number + && a.start_sample == b.start_sample && a.median_before == b.median_before + && a.end_reason == b.end_reason && a.end_reason_forced == b.end_reason_forced + && a.run_info == b.run_info && a.num_minknow_events == b.num_minknow_events + && a.tracked_scaling_scale == b.tracked_scaling_scale + && a.tracked_scaling_shift == b.tracked_scaling_shift + && a.predicted_scaling_scale == b.predicted_scaling_scale + && a.predicted_scaling_shift == b.predicted_scaling_shift + && a.num_reads_since_mux_change == b.num_reads_since_mux_change + && a.time_since_mux_change == b.time_since_mux_change && a.channel == b.channel + && a.well == b.well && a.pore_type == b.pore_type + && a.calibration_offset == b.calibration_offset + && a.calibration_scale == b.calibration_scale; } class RunInfoData { public: using MapType = std::vector>; - RunInfoData(std::string const& acquisition_id, - std::int64_t acquisition_start_time, - std::int16_t adc_max, - std::int16_t adc_min, - MapType const& context_tags, - std::string const& experiment_name, - std::string const& flow_cell_id, - std::string const& flow_cell_product_code, - std::string const& protocol_name, - std::string const& protocol_run_id, - std::int64_t protocol_start_time, - std::string const& sample_id, - std::uint16_t sample_rate, - std::string const& sequencing_kit, - std::string const& sequencer_position, - std::string const& sequencer_position_type, - std::string const& software, - std::string const& system_name, - std::string const& system_type, - MapType const& tracking_id) - : acquisition_id(acquisition_id), - acquisition_start_time(acquisition_start_time), - adc_max(adc_max), - adc_min(adc_min), - context_tags(context_tags), - experiment_name(experiment_name), - flow_cell_id(flow_cell_id), - flow_cell_product_code(flow_cell_product_code), - protocol_name(protocol_name), - protocol_run_id(protocol_run_id), - protocol_start_time(protocol_start_time), - sample_id(sample_id), - sample_rate(sample_rate), - sequencing_kit(sequencing_kit), - sequencer_position(sequencer_position), - sequencer_position_type(sequencer_position_type), - software(software), - system_name(system_name), - system_type(system_type), - tracking_id(tracking_id) {} - static std::int64_t convert_from_system_clock(std::chrono::system_clock::time_point value) { + RunInfoData( + std::string const & acquisition_id, + std::int64_t acquisition_start_time, + std::int16_t adc_max, + std::int16_t adc_min, + MapType const & context_tags, + std::string const & experiment_name, + std::string const & flow_cell_id, + std::string const & flow_cell_product_code, + std::string const & protocol_name, + std::string const & protocol_run_id, + std::int64_t protocol_start_time, + std::string const & sample_id, + std::uint16_t sample_rate, + std::string const & sequencing_kit, + std::string const & sequencer_position, + std::string const & sequencer_position_type, + std::string const & software, + std::string const & system_name, + std::string const & system_type, + MapType const & tracking_id) + : acquisition_id(acquisition_id) + , acquisition_start_time(acquisition_start_time) + , adc_max(adc_max) + , adc_min(adc_min) + , context_tags(context_tags) + , experiment_name(experiment_name) + , flow_cell_id(flow_cell_id) + , flow_cell_product_code(flow_cell_product_code) + , protocol_name(protocol_name) + , protocol_run_id(protocol_run_id) + , protocol_start_time(protocol_start_time) + , sample_id(sample_id) + , sample_rate(sample_rate) + , sequencing_kit(sequencing_kit) + , sequencer_position(sequencer_position) + , sequencer_position_type(sequencer_position_type) + , software(software) + , system_name(system_name) + , system_type(system_type) + , tracking_id(tracking_id) + { + } + + static std::int64_t convert_from_system_clock(std::chrono::system_clock::time_point value) + { return value.time_since_epoch() / std::chrono::milliseconds(1); } static std::chrono::system_clock::time_point convert_to_system_clock( - std::int64_t since_epoch_ms) { + std::int64_t since_epoch_ms) + { return std::chrono::system_clock::time_point() + std::chrono::milliseconds(since_epoch_ms); } @@ -187,19 +198,20 @@ class RunInfoData { MapType tracking_id; }; -inline bool operator==(RunInfoData const& a, RunInfoData const& b) { - return a.acquisition_id == b.acquisition_id && - a.acquisition_start_time == b.acquisition_start_time && a.adc_max == b.adc_max && - a.adc_min == b.adc_min && a.context_tags == b.context_tags && - a.experiment_name == b.experiment_name && a.flow_cell_id == b.flow_cell_id && - a.flow_cell_product_code == b.flow_cell_product_code && - a.protocol_name == b.protocol_name && a.protocol_run_id == b.protocol_run_id && - a.protocol_start_time == b.protocol_start_time && a.sample_id == b.sample_id && - a.sample_rate == b.sample_rate && a.sequencing_kit == b.sequencing_kit && - a.sequencer_position == b.sequencer_position && - a.sequencer_position_type == b.sequencer_position_type && a.software == b.software && - a.system_name == b.system_name && a.system_type == b.system_type && - a.tracking_id == b.tracking_id; +inline bool operator==(RunInfoData const & a, RunInfoData const & b) +{ + return a.acquisition_id == b.acquisition_id + && a.acquisition_start_time == b.acquisition_start_time && a.adc_max == b.adc_max + && a.adc_min == b.adc_min && a.context_tags == b.context_tags + && a.experiment_name == b.experiment_name && a.flow_cell_id == b.flow_cell_id + && a.flow_cell_product_code == b.flow_cell_product_code + && a.protocol_name == b.protocol_name && a.protocol_run_id == b.protocol_run_id + && a.protocol_start_time == b.protocol_start_time && a.sample_id == b.sample_id + && a.sample_rate == b.sample_rate && a.sequencing_kit == b.sequencing_kit + && a.sequencer_position == b.sequencer_position + && a.sequencer_position_type == b.sequencer_position_type && a.software == b.software + && a.system_name == b.system_name && a.system_type == b.system_type + && a.tracking_id == b.tracking_id; } enum class ReadEndReason : std::uint8_t { @@ -213,9 +225,11 @@ enum class ReadEndReason : std::uint8_t { last_end_reason = signal_negative }; -inline char const* end_reason_as_string(ReadEndReason reason) { - static_assert(ReadEndReason::last_end_reason == ReadEndReason::signal_negative, - "Need to add new end reason to this function"); +inline char const * end_reason_as_string(ReadEndReason reason) +{ + static_assert( + ReadEndReason::last_end_reason == ReadEndReason::signal_negative, + "Need to add new end reason to this function"); switch (reason) { case ReadEndReason::mux_change: return "mux_change"; @@ -233,9 +247,11 @@ inline char const* end_reason_as_string(ReadEndReason reason) { return "unknown"; } -inline ReadEndReason end_reason_from_string(std::string const& reason) { - static_assert(ReadEndReason::last_end_reason == ReadEndReason::signal_negative, - "Need to add new end reason to this function"); +inline ReadEndReason end_reason_from_string(std::string const & reason) +{ + static_assert( + ReadEndReason::last_end_reason == ReadEndReason::signal_negative, + "Need to add new end reason to this function"); if (reason == "unknown") { return ReadEndReason::unknown; } else if (reason == "mux_change") { @@ -261,11 +277,11 @@ class POD5_FORMAT_EXPORT ReadIdSearchInput { std::size_t index; }; - ReadIdSearchInput(gsl::span const& input_ids); + ReadIdSearchInput(gsl::span const & input_ids); std::size_t read_id_count() const { return m_search_read_ids.size(); } - InputId const& operator[](std::size_t i) const { return m_search_read_ids[i]; } + InputId const & operator[](std::size_t i) const { return m_search_read_ids[i]; } private: std::vector m_search_read_ids; diff --git a/c++/pod5_format/read_table_writer.cpp b/c++/pod5_format/read_table_writer.cpp index fbc4a60..7102c22 100644 --- a/c++/pod5_format/read_table_writer.cpp +++ b/c++/pod5_format/read_table_writer.cpp @@ -11,35 +11,41 @@ namespace pod5 { -ReadTableWriter::ReadTableWriter(std::shared_ptr&& writer, - std::shared_ptr&& schema, - std::shared_ptr const& field_locations, - std::size_t table_batch_size, - std::shared_ptr const& pore_writer, - std::shared_ptr const& end_reason_writer, - std::shared_ptr const& run_info_writer, - arrow::MemoryPool* pool) - : m_schema(schema), - m_field_locations(field_locations), - m_table_batch_size(table_batch_size), - m_writer(std::move(writer)), - m_field_builders(m_field_locations, pool) { +ReadTableWriter::ReadTableWriter( + std::shared_ptr && writer, + std::shared_ptr && schema, + std::shared_ptr const & field_locations, + std::size_t table_batch_size, + std::shared_ptr const & pore_writer, + std::shared_ptr const & end_reason_writer, + std::shared_ptr const & run_info_writer, + arrow::MemoryPool * pool) +: m_schema(schema) +, m_field_locations(field_locations) +, m_table_batch_size(table_batch_size) +, m_writer(std::move(writer)) +, m_field_builders(m_field_locations, pool) +{ m_field_builders.get_builder(m_field_locations->pore_type).set_dict_writer(pore_writer); m_field_builders.get_builder(m_field_locations->end_reason).set_dict_writer(end_reason_writer); m_field_builders.get_builder(m_field_locations->run_info).set_dict_writer(run_info_writer); } -ReadTableWriter::ReadTableWriter(ReadTableWriter&& other) = default; -ReadTableWriter& ReadTableWriter::operator=(ReadTableWriter&&) = default; -ReadTableWriter::~ReadTableWriter() { +ReadTableWriter::ReadTableWriter(ReadTableWriter && other) = default; +ReadTableWriter & ReadTableWriter::operator=(ReadTableWriter &&) = default; + +ReadTableWriter::~ReadTableWriter() +{ if (m_writer) { (void)close(); } } -Result ReadTableWriter::add_read(ReadData const& read_data, - gsl::span const& signal, - std::uint64_t signal_duration) { +Result ReadTableWriter::add_read( + ReadData const & read_data, + gsl::span const & signal, + std::uint64_t signal_duration) +{ POD5_TRACE_FUNCTION(); if (!m_writer) { return Status::IOError("Writer terminated"); @@ -47,23 +53,34 @@ Result ReadTableWriter::add_read(ReadData const& read_data, auto row_id = m_written_batched_row_count + m_current_batch_row_count; ARROW_RETURN_NOT_OK(m_field_builders.append( - // V0 Fields - read_data.read_id, signal, read_data.read_number, read_data.start_sample, - read_data.median_before, - - // V1 Fields - read_data.num_minknow_events, read_data.tracked_scaling_scale, - read_data.tracked_scaling_shift, read_data.predicted_scaling_scale, - read_data.predicted_scaling_shift, read_data.num_reads_since_mux_change, - read_data.time_since_mux_change, - - // V2 Fields - signal_duration, - - // V3 Fields - read_data.channel, read_data.well, read_data.pore_type, read_data.calibration_offset, - read_data.calibration_scale, read_data.end_reason, read_data.end_reason_forced, - read_data.run_info)); + // V0 Fields + read_data.read_id, + signal, + read_data.read_number, + read_data.start_sample, + read_data.median_before, + + // V1 Fields + read_data.num_minknow_events, + read_data.tracked_scaling_scale, + read_data.tracked_scaling_shift, + read_data.predicted_scaling_scale, + read_data.predicted_scaling_shift, + read_data.num_reads_since_mux_change, + read_data.time_since_mux_change, + + // V2 Fields + signal_duration, + + // V3 Fields + read_data.channel, + read_data.well, + read_data.pore_type, + read_data.calibration_offset, + read_data.calibration_scale, + read_data.end_reason, + read_data.end_reason_forced, + read_data.run_info)); ++m_current_batch_row_count; @@ -73,7 +90,8 @@ Result ReadTableWriter::add_read(ReadData const& read_data, return row_id; } -Status ReadTableWriter::close() { +Status ReadTableWriter::close() +{ // Check for already closed if (!m_writer) { return Status::OK(); @@ -85,7 +103,8 @@ Status ReadTableWriter::close() { return Status::OK(); } -Status ReadTableWriter::write_batch() { +Status ReadTableWriter::write_batch() +{ POD5_TRACE_FUNCTION(); if (m_current_batch_row_count == 0) { return Status::OK(); @@ -98,7 +117,7 @@ Status ReadTableWriter::write_batch() { ARROW_ASSIGN_OR_RAISE(auto columns, m_field_builders.finish_columns()); auto const record_batch = - arrow::RecordBatch::Make(m_schema, m_current_batch_row_count, std::move(columns)); + arrow::RecordBatch::Make(m_schema, m_current_batch_row_count, std::move(columns)); m_written_batched_row_count += m_current_batch_row_count; m_current_batch_row_count = 0; @@ -110,13 +129,14 @@ Status ReadTableWriter::write_batch() { Status ReadTableWriter::reserve_rows() { return m_field_builders.reserve(m_table_batch_size); } Result make_read_table_writer( - std::shared_ptr const& sink, - std::shared_ptr const& metadata, - std::size_t table_batch_size, - std::shared_ptr const& pore_writer, - std::shared_ptr const& end_reason_writer, - std::shared_ptr const& run_info_writer, - arrow::MemoryPool* pool) { + std::shared_ptr const & sink, + std::shared_ptr const & metadata, + std::size_t table_batch_size, + std::shared_ptr const & pore_writer, + std::shared_ptr const & end_reason_writer, + std::shared_ptr const & run_info_writer, + arrow::MemoryPool * pool) +{ auto field_locations = std::make_shared(); auto schema = field_locations->make_writer_schema(metadata); @@ -128,9 +148,15 @@ Result make_read_table_writer( ARROW_ASSIGN_OR_RAISE(auto writer, arrow::ipc::MakeFileWriter(sink, schema, options, metadata)); - auto read_table_writer = - ReadTableWriter(std::move(writer), std::move(schema), field_locations, table_batch_size, - pore_writer, end_reason_writer, run_info_writer, pool); + auto read_table_writer = ReadTableWriter( + std::move(writer), + std::move(schema), + field_locations, + table_batch_size, + pore_writer, + end_reason_writer, + run_info_writer, + pool); ARROW_RETURN_NOT_OK(read_table_writer.reserve_rows()); diff --git a/c++/pod5_format/read_table_writer.h b/c++/pod5_format/read_table_writer.h index f09625e..1f981f2 100644 --- a/c++/pod5_format/read_table_writer.h +++ b/c++/pod5_format/read_table_writer.h @@ -12,9 +12,11 @@ namespace arrow { class Schema; + namespace io { class OutputStream; } + namespace ipc { class RecordBatchWriter; } @@ -24,18 +26,19 @@ namespace pod5 { class POD5_FORMAT_EXPORT ReadTableWriter { public: - ReadTableWriter(std::shared_ptr&& writer, - std::shared_ptr&& schema, - std::shared_ptr const& field_locations, - std::size_t table_batch_size, - std::shared_ptr const& pore_writer, - std::shared_ptr const& end_reason_writer, - std::shared_ptr const& run_info_writer, - arrow::MemoryPool* pool); - ReadTableWriter(ReadTableWriter&&); - ReadTableWriter& operator=(ReadTableWriter&&); - ReadTableWriter(ReadTableWriter const&) = delete; - ReadTableWriter& operator=(ReadTableWriter const&) = delete; + ReadTableWriter( + std::shared_ptr && writer, + std::shared_ptr && schema, + std::shared_ptr const & field_locations, + std::size_t table_batch_size, + std::shared_ptr const & pore_writer, + std::shared_ptr const & end_reason_writer, + std::shared_ptr const & run_info_writer, + arrow::MemoryPool * pool); + ReadTableWriter(ReadTableWriter &&); + ReadTableWriter & operator=(ReadTableWriter &&); + ReadTableWriter(ReadTableWriter const &) = delete; + ReadTableWriter & operator=(ReadTableWriter const &) = delete; ~ReadTableWriter(); /// \brief Add a read to the read table, adding to the current batch. @@ -43,9 +46,10 @@ class POD5_FORMAT_EXPORT ReadTableWriter { /// \param signal List of signal table row indices that belong to this read. /// \param signal_duration The length of the read in samples. /// \returns The row index of the inserted read, or a status on failure. - Result add_read(ReadData const& read_data, - gsl::span const& signal, - std::uint64_t signal_duration); + Result add_read( + ReadData const & read_data, + gsl::span const & signal, + std::uint64_t signal_duration); /// \brief Close this writer, signaling no further data will be written to the writer. Status close(); @@ -76,12 +80,12 @@ class POD5_FORMAT_EXPORT ReadTableWriter { /// \param pool Pool to be used for building table in memory. /// \returns The writer for the new table. POD5_FORMAT_EXPORT Result make_read_table_writer( - std::shared_ptr const& sink, - std::shared_ptr const& metadata, - std::size_t table_batch_size, - std::shared_ptr const& pore_writer, - std::shared_ptr const& end_reason_writer, - std::shared_ptr const& run_info_writer, - arrow::MemoryPool* pool); + std::shared_ptr const & sink, + std::shared_ptr const & metadata, + std::size_t table_batch_size, + std::shared_ptr const & pore_writer, + std::shared_ptr const & end_reason_writer, + std::shared_ptr const & run_info_writer, + arrow::MemoryPool * pool); } // namespace pod5 diff --git a/c++/pod5_format/read_table_writer_utils.cpp b/c++/pod5_format/read_table_writer_utils.cpp index c99b444..9f6abd6 100644 --- a/c++/pod5_format/read_table_writer_utils.cpp +++ b/c++/pod5_format/read_table_writer_utils.cpp @@ -13,22 +13,24 @@ namespace detail { template arrow::Result> get_array_data( - std::shared_ptr const& type, - PrimitiveDictionaryKeyBuilder const& builder, - std::size_t expected_length) { + std::shared_ptr const & type, + PrimitiveDictionaryKeyBuilder const & builder, + std::size_t expected_length) +{ arrow::TypedBufferBuilder buffer_builder; auto data = builder.get_data(); return arrow::ArrayData::Make(type, expected_length, {nullptr, data}, 0); } arrow::Result> get_array_data( - std::shared_ptr const& type, - StringDictionaryKeyBuilder const& builder, - std::size_t expected_length) { + std::shared_ptr const & type, + StringDictionaryKeyBuilder const & builder, + std::size_t expected_length) +{ auto const value_data = builder.get_string_data(); arrow::TypedBufferBuilder offset_builder; - auto const& offset_data = builder.get_typed_offset_data(); + auto const & offset_data = builder.get_typed_offset_data(); if (offset_data.size() != expected_length) { return Status::Invalid("Invalid size for field in struct"); } @@ -43,11 +45,12 @@ arrow::Result> get_array_data( } arrow::Result> get_array_data( - std::shared_ptr const& type, - StringMapDictionaryKeyBuilder const& builder, - std::size_t expected_length) { + std::shared_ptr const & type, + StringMapDictionaryKeyBuilder const & builder, + std::size_t expected_length) +{ arrow::TypedBufferBuilder offset_builder; - auto const& offset_data = builder.get_typed_offset_data(); + auto const & offset_data = builder.get_typed_offset_data(); if (offset_data.size() != expected_length) { return Status::Invalid("Invalid size for field in struct"); } @@ -64,23 +67,22 @@ arrow::Result> get_array_data( } // Extract map keys + values into their own array data: - auto const& map_type = std::static_pointer_cast(type); - auto const& key_type = map_type->key_type(); - auto const& item_type = map_type->item_type(); + auto const & map_type = std::static_pointer_cast(type); + auto const & key_type = map_type->key_type(); + auto const & item_type = map_type->item_type(); - ARROW_ASSIGN_OR_RAISE(auto key_data, - get_array_data(key_type, builder.key_builder(), final_item_length)); - ARROW_ASSIGN_OR_RAISE(auto item_data, - get_array_data(item_type, builder.value_builder(), final_item_length)); + ARROW_ASSIGN_OR_RAISE( + auto key_data, get_array_data(key_type, builder.key_builder(), final_item_length)); + ARROW_ASSIGN_OR_RAISE( + auto item_data, get_array_data(item_type, builder.value_builder(), final_item_length)); // Pack this data out as a struct: - std::shared_ptr items = - arrow::ArrayData::Make(map_type->value_type(), final_item_length, {nullptr, offsets}, - {key_data, item_data}, 0); + std::shared_ptr items = arrow::ArrayData::Make( + map_type->value_type(), final_item_length, {nullptr, offsets}, {key_data, item_data}, 0); // And add this struct to the map/list as the value data, along with the offsets: - auto data = arrow::ArrayData::Make(type, expected_length, {nullptr, offsets}, - {std::move(items)}, 0); + auto data = + arrow::ArrayData::Make(type, expected_length, {nullptr, offsets}, {std::move(items)}, 0); arrow::MapArray array(data); assert(array.length() == (std::int64_t)offset_data.size()); @@ -91,14 +93,16 @@ arrow::Result> get_array_data( } template -arrow::Status do_struct_array_data_unpack(std::vector>& dest, - std::shared_ptr const& type, - std::size_t expected_length, - BuilderTuple const& builders) { +arrow::Status do_struct_array_data_unpack( + std::vector> & dest, + std::shared_ptr const & type, + std::size_t expected_length, + BuilderTuple const & builders) +{ static_assert(CurrentIndex >= 0, "Current index must be greater than zero"); - auto const& field_type = type->field(CurrentIndex)->type(); + auto const & field_type = type->field(CurrentIndex)->type(); - auto const& builder = std::get(builders); + auto const & builder = std::get(builders); ARROW_ASSIGN_OR_RAISE(dest[CurrentIndex], get_array_data(field_type, builder, expected_length)); return Status::OK(); @@ -109,10 +113,12 @@ struct UnpackStructArrayData; template struct UnpackStructArrayData<0, BuilderTuple> { - static arrow::Status unpack(std::vector>& dest, - std::shared_ptr const& type, - std::size_t expected_length, - BuilderTuple const& builders) { + static arrow::Status unpack( + std::vector> & dest, + std::shared_ptr const & type, + std::size_t expected_length, + BuilderTuple const & builders) + { // Dump the last item: return do_struct_array_data_unpack<0>(dest, type, expected_length, builders); } @@ -120,23 +126,26 @@ struct UnpackStructArrayData<0, BuilderTuple> { template struct UnpackStructArrayData { - static arrow::Status unpack(std::vector>& dest, - std::shared_ptr const& type, - std::size_t expected_length, - BuilderTuple const& builders) { + static arrow::Status unpack( + std::vector> & dest, + std::shared_ptr const & type, + std::size_t expected_length, + BuilderTuple const & builders) + { // Dump this builders first: RETURN_NOT_OK( - do_struct_array_data_unpack(dest, type, expected_length, builders)); + do_struct_array_data_unpack(dest, type, expected_length, builders)); // Then recursively dump the other builders: return UnpackStructArrayData::unpack( - dest, type, expected_length, builders); + dest, type, expected_length, builders); } }; template arrow::Result> get_struct_array( - std::shared_ptr const& type, - BuilderTuple const& builders) { + std::shared_ptr const & type, + BuilderTuple const & builders) +{ auto const length = std::get<0>(builders).length(); std::int64_t null_count = 0; @@ -158,11 +167,13 @@ arrow::Result> get_struct_array( } // namespace detail -arrow::Result> make_pore_writer(arrow::MemoryPool* pool) { +arrow::Result> make_pore_writer(arrow::MemoryPool * pool) +{ return std::make_shared(pool); } -arrow::Result> make_end_reason_writer(arrow::MemoryPool* pool) { +arrow::Result> make_end_reason_writer(arrow::MemoryPool * pool) +{ std::shared_ptr end_reasons; { arrow::StringBuilder builder(pool); @@ -176,37 +187,44 @@ arrow::Result> make_end_reason_writer(arrow::Me return std::make_shared(end_reasons); } -arrow::Result> make_run_info_writer(arrow::MemoryPool* pool) { +arrow::Result> make_run_info_writer(arrow::MemoryPool * pool) +{ return std::make_shared(pool); } pod5::Result> DictionaryWriter::build_dictionary_array( - std::shared_ptr const& indices) { + std::shared_ptr const & indices) +{ ARROW_ASSIGN_OR_RAISE(auto res, get_value_array()); return arrow::DictionaryArray::FromArrays(indices, res); } -PoreWriter::PoreWriter(arrow::MemoryPool* pool) : m_builder(pool) {} +PoreWriter::PoreWriter(arrow::MemoryPool * pool) : m_builder(pool) {} -pod5::Result> PoreWriter::get_value_array() { +pod5::Result> PoreWriter::get_value_array() +{ ARROW_ASSIGN_OR_RAISE(auto array_data, get_array_data(arrow::utf8(), m_builder, item_count())); return std::make_shared(array_data); } std::size_t PoreWriter::item_count() { return m_builder.length(); } -EndReasonWriter::EndReasonWriter(std::shared_ptr const& end_reasons) - : m_end_reasons(end_reasons) {} +EndReasonWriter::EndReasonWriter(std::shared_ptr const & end_reasons) +: m_end_reasons(end_reasons) +{ +} -pod5::Result> EndReasonWriter::get_value_array() { +pod5::Result> EndReasonWriter::get_value_array() +{ return m_end_reasons; } std::size_t EndReasonWriter::item_count() { return m_end_reasons->length(); } -RunInfoWriter::RunInfoWriter(arrow::MemoryPool* pool) : m_builder(pool) {} +RunInfoWriter::RunInfoWriter(arrow::MemoryPool * pool) : m_builder(pool) {} -pod5::Result> RunInfoWriter::get_value_array() { +pod5::Result> RunInfoWriter::get_value_array() +{ ARROW_ASSIGN_OR_RAISE(auto array_data, get_array_data(arrow::utf8(), m_builder, item_count())); return std::make_shared(array_data); } diff --git a/c++/pod5_format/read_table_writer_utils.h b/c++/pod5_format/read_table_writer_utils.h index fafeb88..9bfa6e1 100644 --- a/c++/pod5_format/read_table_writer_utils.h +++ b/c++/pod5_format/read_table_writer_utils.h @@ -23,11 +23,12 @@ namespace detail { template class PrimitiveDictionaryKeyBuilder { public: - arrow::Status init_buffer(arrow::MemoryPool* pool) { return m_values.init_buffer(pool); } + arrow::Status init_buffer(arrow::MemoryPool * pool) { return m_values.init_buffer(pool); } - arrow::Status append(T const& value) { return m_values.append(value); } + arrow::Status append(T const & value) { return m_values.append(value); } std::size_t length() const { return m_values.size(); } + std::shared_ptr get_data() const { return m_values.get_buffer(); } private: @@ -38,9 +39,10 @@ class PrimitiveDictionaryKeyBuilder { template <> class PrimitiveDictionaryKeyBuilder { public: - arrow::Status init_buffer(arrow::MemoryPool* pool) { return m_values.init_buffer(pool); } + arrow::Status init_buffer(arrow::MemoryPool * pool) { return m_values.init_buffer(pool); } - arrow::Status append(bool value) { + arrow::Status append(bool value) + { ARROW_RETURN_NOT_OK(m_values.resize((m_bit_length / 8) + 1)); auto mutable_data = m_values.mutable_data(); arrow::bit_util::SetBitTo(mutable_data, m_bit_length, value); @@ -50,6 +52,7 @@ class PrimitiveDictionaryKeyBuilder { } std::size_t length() const { return m_bit_length; } + std::shared_ptr get_data() const { return m_values.get_buffer(); } private: @@ -59,24 +62,31 @@ class PrimitiveDictionaryKeyBuilder { class StringDictionaryKeyBuilder { public: - StringDictionaryKeyBuilder(arrow::MemoryPool* pool = nullptr) - : m_offset_values(pool), m_string_values(pool) {} + StringDictionaryKeyBuilder(arrow::MemoryPool * pool = nullptr) + : m_offset_values(pool) + , m_string_values(pool) + { + } - arrow::Status init_buffer(arrow::MemoryPool* pool) { + arrow::Status init_buffer(arrow::MemoryPool * pool) + { ARROW_RETURN_NOT_OK(m_offset_values.init_buffer(pool)); return m_string_values.init_buffer(pool); } - arrow::Status append(std::string const& value) { + arrow::Status append(std::string const & value) + { ARROW_RETURN_NOT_OK(m_offset_values.append(m_string_values.size())); return m_string_values.append_array( - gsl::make_span(value.data(), value.size()).as_span()); + gsl::make_span(value.data(), value.size()).as_span()); } std::size_t length() const { return m_offset_values.size(); } std::shared_ptr get_string_data() const { return m_string_values.get_buffer(); } - gsl::span get_typed_offset_data() const { + + gsl::span get_typed_offset_data() const + { return m_offset_values.get_data_span(); } @@ -87,15 +97,17 @@ class StringDictionaryKeyBuilder { class StringMapDictionaryKeyBuilder { public: - arrow::Status init_buffer(arrow::MemoryPool* pool) { + arrow::Status init_buffer(arrow::MemoryPool * pool) + { ARROW_RETURN_NOT_OK(m_offset_values.init_buffer(pool)); ARROW_RETURN_NOT_OK(m_key_builder.init_buffer(pool)); return m_value_builder.init_buffer(pool); } - arrow::Status append(pod5::RunInfoData::MapType const& value) { + arrow::Status append(pod5::RunInfoData::MapType const & value) + { ARROW_RETURN_NOT_OK(m_offset_values.append(m_key_builder.length())); - for (auto const& item : value) { + for (auto const & item : value) { ARROW_RETURN_NOT_OK(m_key_builder.append(item.first)); ARROW_RETURN_NOT_OK(m_value_builder.append(item.second)); } @@ -104,9 +116,12 @@ class StringMapDictionaryKeyBuilder { std::size_t length() const { return m_offset_values.size(); } - StringDictionaryKeyBuilder const& key_builder() const { return m_key_builder; } - StringDictionaryKeyBuilder const& value_builder() const { return m_value_builder; } - gsl::span get_typed_offset_data() const { + StringDictionaryKeyBuilder const & key_builder() const { return m_key_builder; } + + StringDictionaryKeyBuilder const & value_builder() const { return m_value_builder; } + + gsl::span get_typed_offset_data() const + { return m_offset_values.get_data_span(); } @@ -117,19 +132,20 @@ class StringMapDictionaryKeyBuilder { }; template -arrow::Result unpack_struct_builder_args(BuilderTuple& builders, Arg&& arg) { - auto& builder = std::get(builders); +arrow::Result unpack_struct_builder_args(BuilderTuple & builders, Arg && arg) +{ + auto & builder = std::get(builders); auto index = builder.length(); ARROW_RETURN_NOT_OK(builder.append(arg)); return index; } template -arrow::Result unpack_struct_builder_args(BuilderTuple& builder, - FirstArg&& first_arg, - Args&&... args) { +arrow::Result +unpack_struct_builder_args(BuilderTuple & builder, FirstArg && first_arg, Args &&... args) +{ ARROW_RETURN_NOT_OK(unpack_struct_builder_args(builder, first_arg)); - return unpack_struct_builder_args(builder, std::forward(args)...); + return unpack_struct_builder_args(builder, std::forward(args)...); } } // namespace detail @@ -137,17 +153,19 @@ arrow::Result unpack_struct_builder_args(BuilderTuple& builder, template class StructBuilder { public: - StructBuilder(arrow::MemoryPool* pool) { - detail::for_each_in_tuple(m_builders, - [&](auto& x, std::size_t _) { (void)x.init_buffer(pool); }); + StructBuilder(arrow::MemoryPool * pool) + { + detail::for_each_in_tuple( + m_builders, [&](auto & x, std::size_t _) { (void)x.init_buffer(pool); }); } template - arrow::Result append(Args&&... args) { - return detail::unpack_struct_builder_args<0>(m_builders, std::forward(args)...); + arrow::Result append(Args &&... args) + { + return detail::unpack_struct_builder_args<0>(m_builders, std::forward(args)...); } - std::tuple& builders() { return m_builders; } + std::tuple & builders() { return m_builders; } private: std::tuple m_builders; @@ -155,9 +173,10 @@ class StructBuilder { class POD5_FORMAT_EXPORT PoreWriter : public DictionaryWriter { public: - PoreWriter(arrow::MemoryPool* pool); + PoreWriter(arrow::MemoryPool * pool); - pod5::Result add(std::string const& pore_type) { + pod5::Result add(std::string const & pore_type) + { auto const index = item_count(); ARROW_RETURN_NOT_OK(m_builder.append(pore_type)); return index; @@ -172,9 +191,10 @@ class POD5_FORMAT_EXPORT PoreWriter : public DictionaryWriter { class POD5_FORMAT_EXPORT EndReasonWriter : public DictionaryWriter { public: - EndReasonWriter(std::shared_ptr const& end_reasons); + EndReasonWriter(std::shared_ptr const & end_reasons); - pod5::Result lookup(ReadEndReason end_reason) const { + pod5::Result lookup(ReadEndReason end_reason) const + { if (end_reason > ReadEndReason::last_end_reason) { return pod5::Status::Invalid("Invalid read end reason requested"); } @@ -190,9 +210,10 @@ class POD5_FORMAT_EXPORT EndReasonWriter : public DictionaryWriter { class POD5_FORMAT_EXPORT RunInfoWriter : public DictionaryWriter { public: - RunInfoWriter(arrow::MemoryPool* pool); + RunInfoWriter(arrow::MemoryPool * pool); - pod5::Result add(std::string const& acquisition_id) { + pod5::Result add(std::string const & acquisition_id) + { auto const index = item_count(); ARROW_RETURN_NOT_OK(m_builder.append(acquisition_id)); return index; @@ -206,12 +227,12 @@ class POD5_FORMAT_EXPORT RunInfoWriter : public DictionaryWriter { }; POD5_FORMAT_EXPORT arrow::Result> make_pore_writer( - arrow::MemoryPool* pool); + arrow::MemoryPool * pool); POD5_FORMAT_EXPORT arrow::Result> make_end_reason_writer( - arrow::MemoryPool* pool); + arrow::MemoryPool * pool); POD5_FORMAT_EXPORT arrow::Result> make_run_info_writer( - arrow::MemoryPool* pool); + arrow::MemoryPool * pool); } // namespace pod5 diff --git a/c++/pod5_format/run_info_table_reader.cpp b/c++/pod5_format/run_info_table_reader.cpp index 66d3abb..dd650ca 100644 --- a/c++/pod5_format/run_info_table_reader.cpp +++ b/c++/pod5_format/run_info_table_reader.cpp @@ -14,13 +14,14 @@ namespace pod5 { inline std::vector> value_for_map( - std::shared_ptr const& map_array, - std::size_t row_index) { + std::shared_ptr const & map_array, + std::size_t row_index) +{ std::size_t offset = map_array->value_offset(row_index); std::size_t length = map_array->value_length(row_index); - auto const& keys = std::dynamic_pointer_cast(map_array->keys()); - auto const& items = std::dynamic_pointer_cast(map_array->items()); + auto const & keys = std::dynamic_pointer_cast(map_array->keys()); + auto const & items = std::dynamic_pointer_cast(map_array->items()); std::vector> result; for (std::size_t i = offset; i < offset + length; ++i) { @@ -30,28 +31,34 @@ inline std::vector> value_for_map( } RunInfoTableRecordBatch::RunInfoTableRecordBatch( - std::shared_ptr&& batch, - std::shared_ptr const& field_locations) - : TableRecordBatch(std::move(batch)), m_field_locations(field_locations) {} + std::shared_ptr && batch, + std::shared_ptr const & field_locations) +: TableRecordBatch(std::move(batch)) +, m_field_locations(field_locations) +{ +} -RunInfoTableRecordBatch::RunInfoTableRecordBatch(RunInfoTableRecordBatch&& other) - : TableRecordBatch(std::move(other)) { +RunInfoTableRecordBatch::RunInfoTableRecordBatch(RunInfoTableRecordBatch && other) +: TableRecordBatch(std::move(other)) +{ m_field_locations = std::move(other.m_field_locations); } -RunInfoTableRecordBatch& RunInfoTableRecordBatch::operator=(RunInfoTableRecordBatch&& other) { - TableRecordBatch& base = *this; +RunInfoTableRecordBatch & RunInfoTableRecordBatch::operator=(RunInfoTableRecordBatch && other) +{ + TableRecordBatch & base = *this; base = other; m_field_locations = std::move(other.m_field_locations); return *this; } -Result RunInfoTableRecordBatch::columns() const { +Result RunInfoTableRecordBatch::columns() const +{ RunInfoTableRecordColumns result; result.table_version = m_field_locations->table_version(); - auto const& bat = batch(); + auto const & bat = batch(); // V0 fields: result.acquisition_id = find_column(bat, m_field_locations->acquisition_id); @@ -81,31 +88,39 @@ Result RunInfoTableRecordBatch::columns() const { //--------------------------------------------------------------------------------------------------------------------- RunInfoTableReader::RunInfoTableReader( - std::shared_ptr&& input_source, - std::shared_ptr&& reader, - std::shared_ptr const& field_locations, - SchemaMetadataDescription&& schema_metadata, - arrow::MemoryPool* pool) - : TableReader(std::move(input_source), std::move(reader), std::move(schema_metadata), pool), - m_field_locations(field_locations) {} - -RunInfoTableReader::RunInfoTableReader(RunInfoTableReader&& other) - : TableReader(std::move(other)), m_field_locations(std::move(other.m_field_locations)) {} - -RunInfoTableReader& RunInfoTableReader::operator=(RunInfoTableReader&& other) { - static_cast(*this) = std::move(static_cast(*this)); + std::shared_ptr && input_source, + std::shared_ptr && reader, + std::shared_ptr const & field_locations, + SchemaMetadataDescription && schema_metadata, + arrow::MemoryPool * pool) +: TableReader(std::move(input_source), std::move(reader), std::move(schema_metadata), pool) +, m_field_locations(field_locations) +{ +} + +RunInfoTableReader::RunInfoTableReader(RunInfoTableReader && other) +: TableReader(std::move(other)) +, m_field_locations(std::move(other.m_field_locations)) +{ +} + +RunInfoTableReader & RunInfoTableReader::operator=(RunInfoTableReader && other) +{ + static_cast(*this) = std::move(static_cast(*this)); m_field_locations = std::move(other.m_field_locations); return *this; } -Result RunInfoTableReader::read_record_batch(std::size_t i) const { +Result RunInfoTableReader::read_record_batch(std::size_t i) const +{ std::lock_guard l(m_batch_get_mutex); ARROW_ASSIGN_OR_RAISE(auto record_batch, reader()->ReadRecordBatch(i)); return RunInfoTableRecordBatch{std::move(record_batch), m_field_locations}; } Result> RunInfoTableReader::find_run_info( - std::string const& acquisition_id) const { + std::string const & acquisition_id) const +{ std::lock_guard l(m_run_info_lookup_mutex); auto it = m_run_info_lookup.find(acquisition_id); if (it != m_run_info_lookup.end()) { @@ -122,23 +137,26 @@ Result> RunInfoTableReader::find_run_info( ARROW_ASSIGN_OR_RAISE(auto columns, batch.columns()); run_info = std::make_shared( - columns.acquisition_id->Value(j).to_string(), - columns.acquisition_start_time->Value(j), columns.adc_max->Value(j), - columns.adc_min->Value(j), value_for_map(columns.context_tags, j), - columns.experiment_name->Value(j).to_string(), - columns.flow_cell_id->Value(j).to_string(), - columns.flow_cell_product_code->Value(j).to_string(), - columns.protocol_name->Value(j).to_string(), - columns.protocol_run_id->Value(j).to_string(), - columns.protocol_start_time->Value(j), - columns.sample_id->Value(j).to_string(), columns.sample_rate->Value(j), - columns.sequencing_kit->Value(j).to_string(), - columns.sequencer_position->Value(j).to_string(), - columns.sequencer_position_type->Value(j).to_string(), - columns.software->Value(j).to_string(), - columns.system_name->Value(j).to_string(), - columns.system_type->Value(j).to_string(), - value_for_map(columns.tracking_id, j)); + columns.acquisition_id->Value(j).to_string(), + columns.acquisition_start_time->Value(j), + columns.adc_max->Value(j), + columns.adc_min->Value(j), + value_for_map(columns.context_tags, j), + columns.experiment_name->Value(j).to_string(), + columns.flow_cell_id->Value(j).to_string(), + columns.flow_cell_product_code->Value(j).to_string(), + columns.protocol_name->Value(j).to_string(), + columns.protocol_run_id->Value(j).to_string(), + columns.protocol_start_time->Value(j), + columns.sample_id->Value(j).to_string(), + columns.sample_rate->Value(j), + columns.sequencing_kit->Value(j).to_string(), + columns.sequencer_position->Value(j).to_string(), + columns.sequencer_position_type->Value(j).to_string(), + columns.software->Value(j).to_string(), + columns.system_name->Value(j).to_string(), + columns.system_type->Value(j).to_string(), + value_for_map(columns.tracking_id, j)); break; } @@ -150,8 +168,8 @@ Result> RunInfoTableReader::find_run_info( } if (!run_info) { - return arrow::Status::Invalid("Failed to find acquisition id '", acquisition_id, - "' in run info table"); + return arrow::Status::Invalid( + "Failed to find acquisition id '", acquisition_id, "' in run info table"); } m_run_info_lookup[acquisition_id] = run_info; @@ -161,8 +179,9 @@ Result> RunInfoTableReader::find_run_info( //--------------------------------------------------------------------------------------------------------------------- Result make_run_info_table_reader( - std::shared_ptr const& input, - arrow::MemoryPool* pool) { + std::shared_ptr const & input, + arrow::MemoryPool * pool) +{ arrow::ipc::IpcReadOptions options; options.memory_pool = pool; @@ -172,13 +191,13 @@ Result make_run_info_table_reader( if (!read_metadata_key_values) { return Status::IOError("Missing metadata on run info table schema"); } - ARROW_ASSIGN_OR_RAISE(auto read_metadata, - read_schema_key_value_metadata(read_metadata_key_values)); - ARROW_ASSIGN_OR_RAISE(auto field_locations, - read_run_info_table_schema(read_metadata, reader->schema())); + ARROW_ASSIGN_OR_RAISE( + auto read_metadata, read_schema_key_value_metadata(read_metadata_key_values)); + ARROW_ASSIGN_OR_RAISE( + auto field_locations, read_run_info_table_schema(read_metadata, reader->schema())); - return RunInfoTableReader({input}, std::move(reader), field_locations, std::move(read_metadata), - pool); + return RunInfoTableReader( + {input}, std::move(reader), field_locations, std::move(read_metadata), pool); } } // namespace pod5 diff --git a/c++/pod5_format/run_info_table_reader.h b/c++/pod5_format/run_info_table_reader.h index 4d054bc..f9a70b4 100644 --- a/c++/pod5_format/run_info_table_reader.h +++ b/c++/pod5_format/run_info_table_reader.h @@ -17,9 +17,11 @@ namespace arrow { class Schema; + namespace io { class RandomAccessFile; } + namespace ipc { class RecordBatchFileReader; } @@ -56,10 +58,10 @@ struct RunInfoTableRecordColumns { class POD5_FORMAT_EXPORT RunInfoTableRecordBatch : public TableRecordBatch { public: RunInfoTableRecordBatch( - std::shared_ptr&& batch, - std::shared_ptr const& field_locations); - RunInfoTableRecordBatch(RunInfoTableRecordBatch&&); - RunInfoTableRecordBatch& operator=(RunInfoTableRecordBatch&&); + std::shared_ptr && batch, + std::shared_ptr const & field_locations); + RunInfoTableRecordBatch(RunInfoTableRecordBatch &&); + RunInfoTableRecordBatch & operator=(RunInfoTableRecordBatch &&); Result columns() const; @@ -69,19 +71,20 @@ class POD5_FORMAT_EXPORT RunInfoTableRecordBatch : public TableRecordBatch { class POD5_FORMAT_EXPORT RunInfoTableReader : public TableReader { public: - RunInfoTableReader(std::shared_ptr&& input_source, - std::shared_ptr&& reader, - std::shared_ptr const& field_locations, - SchemaMetadataDescription&& schema_metadata, - arrow::MemoryPool* pool); + RunInfoTableReader( + std::shared_ptr && input_source, + std::shared_ptr && reader, + std::shared_ptr const & field_locations, + SchemaMetadataDescription && schema_metadata, + arrow::MemoryPool * pool); - RunInfoTableReader(RunInfoTableReader&& other); - RunInfoTableReader& operator=(RunInfoTableReader&& other); + RunInfoTableReader(RunInfoTableReader && other); + RunInfoTableReader & operator=(RunInfoTableReader && other); Result read_record_batch(std::size_t i) const; Result> find_run_info( - std::string const& acquisition_id) const; + std::string const & acquisition_id) const; private: std::shared_ptr m_field_locations; @@ -91,7 +94,7 @@ class POD5_FORMAT_EXPORT RunInfoTableReader : public TableReader { }; POD5_FORMAT_EXPORT Result make_run_info_table_reader( - std::shared_ptr const& sink, - arrow::MemoryPool* pool); + std::shared_ptr const & sink, + arrow::MemoryPool * pool); } // namespace pod5 diff --git a/c++/pod5_format/run_info_table_schema.cpp b/c++/pod5_format/run_info_table_schema.cpp index 4a8dbf4..77d77fa 100644 --- a/c++/pod5_format/run_info_table_schema.cpp +++ b/c++/pod5_format/run_info_table_schema.cpp @@ -6,62 +6,68 @@ namespace pod5 { RunInfoTableSchemaDescription::RunInfoTableSchemaDescription() - : SchemaDescriptionBase(table_version_from_file_version(current_build_version_number())) - // V0 Fields - , - acquisition_id(this, "acquisition_id", arrow::utf8(), RunInfoTableSpecVersion::v0()), - acquisition_start_time(this, - "acquisition_start_time", - arrow::timestamp(arrow::TimeUnit::MILLI, "UTC"), - RunInfoTableSpecVersion::v0()), - adc_max(this, "adc_max", arrow::int16(), RunInfoTableSpecVersion::v0()), - adc_min(this, "adc_min", arrow::int16(), RunInfoTableSpecVersion::v0()), - context_tags(this, - "context_tags", - arrow::map(arrow::utf8(), arrow::utf8()), - RunInfoTableSpecVersion::v0()), - experiment_name(this, "experiment_name", arrow::utf8(), RunInfoTableSpecVersion::v0()), - flow_cell_id(this, "flow_cell_id", arrow::utf8(), RunInfoTableSpecVersion::v0()), - flow_cell_product_code(this, - "flow_cell_product_code", - arrow::utf8(), - RunInfoTableSpecVersion::v0()), - protocol_name(this, "protocol_name", arrow::utf8(), RunInfoTableSpecVersion::v0()), - protocol_run_id(this, "protocol_run_id", arrow::utf8(), RunInfoTableSpecVersion::v0()), - protocol_start_time(this, - "protocol_start_time", - arrow::timestamp(arrow::TimeUnit::MILLI, "UTC"), - RunInfoTableSpecVersion::v0()), - sample_id(this, "sample_id", arrow::utf8(), RunInfoTableSpecVersion::v0()), - sample_rate(this, "sample_rate", arrow::uint16(), RunInfoTableSpecVersion::v0()), - sequencing_kit(this, "sequencing_kit", arrow::utf8(), RunInfoTableSpecVersion::v0()), - sequencer_position(this, - "sequencer_position", - arrow::utf8(), - RunInfoTableSpecVersion::v0()), - sequencer_position_type(this, - "sequencer_position_type", - arrow::utf8(), - RunInfoTableSpecVersion::v0()), - software(this, "software", arrow::utf8(), RunInfoTableSpecVersion::v0()), - system_name(this, "system_name", arrow::utf8(), RunInfoTableSpecVersion::v0()), - system_type(this, "system_type", arrow::utf8(), RunInfoTableSpecVersion::v0()), - tracking_id(this, - "tracking_id", - arrow::map(arrow::utf8(), arrow::utf8()), - RunInfoTableSpecVersion::v0()) {} +: SchemaDescriptionBase(table_version_from_file_version(current_build_version_number())) +// V0 Fields +, acquisition_id(this, "acquisition_id", arrow::utf8(), RunInfoTableSpecVersion::v0()) +, acquisition_start_time( + this, + "acquisition_start_time", + arrow::timestamp(arrow::TimeUnit::MILLI, "UTC"), + RunInfoTableSpecVersion::v0()) +, adc_max(this, "adc_max", arrow::int16(), RunInfoTableSpecVersion::v0()) +, adc_min(this, "adc_min", arrow::int16(), RunInfoTableSpecVersion::v0()) +, context_tags( + this, + "context_tags", + arrow::map(arrow::utf8(), arrow::utf8()), + RunInfoTableSpecVersion::v0()) +, experiment_name(this, "experiment_name", arrow::utf8(), RunInfoTableSpecVersion::v0()) +, flow_cell_id(this, "flow_cell_id", arrow::utf8(), RunInfoTableSpecVersion::v0()) +, flow_cell_product_code( + this, + "flow_cell_product_code", + arrow::utf8(), + RunInfoTableSpecVersion::v0()) +, protocol_name(this, "protocol_name", arrow::utf8(), RunInfoTableSpecVersion::v0()) +, protocol_run_id(this, "protocol_run_id", arrow::utf8(), RunInfoTableSpecVersion::v0()) +, protocol_start_time( + this, + "protocol_start_time", + arrow::timestamp(arrow::TimeUnit::MILLI, "UTC"), + RunInfoTableSpecVersion::v0()) +, sample_id(this, "sample_id", arrow::utf8(), RunInfoTableSpecVersion::v0()) +, sample_rate(this, "sample_rate", arrow::uint16(), RunInfoTableSpecVersion::v0()) +, sequencing_kit(this, "sequencing_kit", arrow::utf8(), RunInfoTableSpecVersion::v0()) +, sequencer_position(this, "sequencer_position", arrow::utf8(), RunInfoTableSpecVersion::v0()) +, sequencer_position_type( + this, + "sequencer_position_type", + arrow::utf8(), + RunInfoTableSpecVersion::v0()) +, software(this, "software", arrow::utf8(), RunInfoTableSpecVersion::v0()) +, system_name(this, "system_name", arrow::utf8(), RunInfoTableSpecVersion::v0()) +, system_type(this, "system_type", arrow::utf8(), RunInfoTableSpecVersion::v0()) +, tracking_id( + this, + "tracking_id", + arrow::map(arrow::utf8(), arrow::utf8()), + RunInfoTableSpecVersion::v0()) +{ +} TableSpecVersion RunInfoTableSchemaDescription::table_version_from_file_version( - Version file_version) const { + Version file_version) const +{ return RunInfoTableSpecVersion::latest(); } Result> read_run_info_table_schema( - SchemaMetadataDescription const& schema_metadata, - std::shared_ptr const& schema) { + SchemaMetadataDescription const & schema_metadata, + std::shared_ptr const & schema) +{ auto result = std::make_shared(); ARROW_RETURN_NOT_OK( - RunInfoTableSchemaDescription::read_schema(result, schema_metadata, schema)); + RunInfoTableSchemaDescription::read_schema(result, schema_metadata, schema)); return result; } diff --git a/c++/pod5_format/run_info_table_schema.h b/c++/pod5_format/run_info_table_schema.h index cc51ab9..14e4f3b 100644 --- a/c++/pod5_format/run_info_table_schema.h +++ b/c++/pod5_format/run_info_table_schema.h @@ -30,8 +30,8 @@ class RunInfoTableSchemaDescription : public SchemaDescriptionBase { public: RunInfoTableSchemaDescription(); - RunInfoTableSchemaDescription(RunInfoTableSchemaDescription const&) = delete; - RunInfoTableSchemaDescription& operator=(RunInfoTableSchemaDescription const&) = delete; + RunInfoTableSchemaDescription(RunInfoTableSchemaDescription const &) = delete; + RunInfoTableSchemaDescription & operator=(RunInfoTableSchemaDescription const &) = delete; TableSpecVersion table_version_from_file_version(Version file_version) const override; @@ -59,31 +59,32 @@ class RunInfoTableSchemaDescription : public SchemaDescriptionBase { // Field Builders only for fields we write in newly generated files. // Should not include fields which are removed in the latest version: using FieldBuilders = FieldBuilder< - // V0 fields - decltype(acquisition_id), - decltype(acquisition_start_time), - decltype(adc_max), - decltype(adc_min), - decltype(context_tags), - decltype(experiment_name), - decltype(flow_cell_id), - decltype(flow_cell_product_code), - decltype(protocol_name), - decltype(protocol_run_id), - decltype(protocol_start_time), - decltype(sample_id), - decltype(sample_rate), - decltype(sequencing_kit), - decltype(sequencer_position), - decltype(sequencer_position_type), - decltype(software), - decltype(system_name), - decltype(system_type), - decltype(tracking_id)>; + // V0 fields + decltype(acquisition_id), + decltype(acquisition_start_time), + decltype(adc_max), + decltype(adc_min), + decltype(context_tags), + decltype(experiment_name), + decltype(flow_cell_id), + decltype(flow_cell_product_code), + decltype(protocol_name), + decltype(protocol_run_id), + decltype(protocol_start_time), + decltype(sample_id), + decltype(sample_rate), + decltype(sequencing_kit), + decltype(sequencer_position), + decltype(sequencer_position_type), + decltype(software), + decltype(system_name), + decltype(system_type), + decltype(tracking_id)>; }; POD5_FORMAT_EXPORT Result> -read_run_info_table_schema(SchemaMetadataDescription const& schema_metadata, - std::shared_ptr const&); +read_run_info_table_schema( + SchemaMetadataDescription const & schema_metadata, + std::shared_ptr const &); } // namespace pod5 diff --git a/c++/pod5_format/run_info_table_writer.cpp b/c++/pod5_format/run_info_table_writer.cpp index 767eb75..286bf06 100644 --- a/c++/pod5_format/run_info_table_writer.cpp +++ b/c++/pod5_format/run_info_table_writer.cpp @@ -13,26 +13,31 @@ namespace pod5 { RunInfoTableWriter::RunInfoTableWriter( - std::shared_ptr&& writer, - std::shared_ptr&& schema, - std::shared_ptr const& field_locations, - std::size_t table_batch_size, - arrow::MemoryPool* pool) - : m_schema(schema), - m_field_locations(field_locations), - m_table_batch_size(table_batch_size), - m_writer(std::move(writer)), - m_field_builders(m_field_locations, pool) {} - -RunInfoTableWriter::RunInfoTableWriter(RunInfoTableWriter&& other) = default; -RunInfoTableWriter& RunInfoTableWriter::operator=(RunInfoTableWriter&&) = default; -RunInfoTableWriter::~RunInfoTableWriter() { + std::shared_ptr && writer, + std::shared_ptr && schema, + std::shared_ptr const & field_locations, + std::size_t table_batch_size, + arrow::MemoryPool * pool) +: m_schema(schema) +, m_field_locations(field_locations) +, m_table_batch_size(table_batch_size) +, m_writer(std::move(writer)) +, m_field_builders(m_field_locations, pool) +{ +} + +RunInfoTableWriter::RunInfoTableWriter(RunInfoTableWriter && other) = default; +RunInfoTableWriter & RunInfoTableWriter::operator=(RunInfoTableWriter &&) = default; + +RunInfoTableWriter::~RunInfoTableWriter() +{ if (m_writer) { (void)close(); } } -Result RunInfoTableWriter::add_run_info(RunInfoData const& run_info_data) { +Result RunInfoTableWriter::add_run_info(RunInfoData const & run_info_data) +{ POD5_TRACE_FUNCTION(); if (!m_writer) { return Status::IOError("Writer terminated"); @@ -40,16 +45,27 @@ Result RunInfoTableWriter::add_run_info(RunInfoData const& run_info auto row_id = m_written_batched_row_count + m_current_batch_row_count; ARROW_RETURN_NOT_OK(m_field_builders.append( - // V0 Fields - run_info_data.acquisition_id, run_info_data.acquisition_start_time, - run_info_data.adc_max, run_info_data.adc_min, run_info_data.context_tags, - run_info_data.experiment_name, run_info_data.flow_cell_id, - run_info_data.flow_cell_product_code, run_info_data.protocol_name, - run_info_data.protocol_run_id, run_info_data.protocol_start_time, - run_info_data.sample_id, run_info_data.sample_rate, run_info_data.sequencing_kit, - run_info_data.sequencer_position, run_info_data.sequencer_position_type, - run_info_data.software, run_info_data.system_name, run_info_data.system_type, - run_info_data.tracking_id)); + // V0 Fields + run_info_data.acquisition_id, + run_info_data.acquisition_start_time, + run_info_data.adc_max, + run_info_data.adc_min, + run_info_data.context_tags, + run_info_data.experiment_name, + run_info_data.flow_cell_id, + run_info_data.flow_cell_product_code, + run_info_data.protocol_name, + run_info_data.protocol_run_id, + run_info_data.protocol_start_time, + run_info_data.sample_id, + run_info_data.sample_rate, + run_info_data.sequencing_kit, + run_info_data.sequencer_position, + run_info_data.sequencer_position_type, + run_info_data.software, + run_info_data.system_name, + run_info_data.system_type, + run_info_data.tracking_id)); ++m_current_batch_row_count; @@ -59,7 +75,8 @@ Result RunInfoTableWriter::add_run_info(RunInfoData const& run_info return row_id; } -Status RunInfoTableWriter::close() { +Status RunInfoTableWriter::close() +{ // Check for already closed if (!m_writer) { return Status::OK(); @@ -71,7 +88,8 @@ Status RunInfoTableWriter::close() { return Status::OK(); } -Status RunInfoTableWriter::write_batch() { +Status RunInfoTableWriter::write_batch() +{ POD5_TRACE_FUNCTION(); if (m_current_batch_row_count == 0) { return Status::OK(); @@ -84,7 +102,7 @@ Status RunInfoTableWriter::write_batch() { ARROW_ASSIGN_OR_RAISE(auto columns, m_field_builders.finish_columns()); auto const record_batch = - arrow::RecordBatch::Make(m_schema, m_current_batch_row_count, std::move(columns)); + arrow::RecordBatch::Make(m_schema, m_current_batch_row_count, std::move(columns)); m_written_batched_row_count += m_current_batch_row_count; m_current_batch_row_count = 0; @@ -96,10 +114,11 @@ Status RunInfoTableWriter::write_batch() { Status RunInfoTableWriter::reserve_rows() { return m_field_builders.reserve(m_table_batch_size); } Result make_run_info_table_writer( - std::shared_ptr const& sink, - std::shared_ptr const& metadata, - std::size_t table_batch_size, - arrow::MemoryPool* pool) { + std::shared_ptr const & sink, + std::shared_ptr const & metadata, + std::size_t table_batch_size, + arrow::MemoryPool * pool) +{ auto field_locations = std::make_shared(); auto schema = field_locations->make_writer_schema(metadata); @@ -108,8 +127,8 @@ Result make_run_info_table_writer( ARROW_ASSIGN_OR_RAISE(auto writer, arrow::ipc::MakeFileWriter(sink, schema, options, metadata)); - auto run_info_table_writer = RunInfoTableWriter(std::move(writer), std::move(schema), - field_locations, table_batch_size, pool); + auto run_info_table_writer = RunInfoTableWriter( + std::move(writer), std::move(schema), field_locations, table_batch_size, pool); ARROW_RETURN_NOT_OK(run_info_table_writer.reserve_rows()); diff --git a/c++/pod5_format/run_info_table_writer.h b/c++/pod5_format/run_info_table_writer.h index f7bd464..3188cca 100644 --- a/c++/pod5_format/run_info_table_writer.h +++ b/c++/pod5_format/run_info_table_writer.h @@ -11,9 +11,11 @@ namespace arrow { class Schema; + namespace io { class OutputStream; } + namespace ipc { class RecordBatchWriter; } @@ -25,21 +27,22 @@ class RunInfoData; class POD5_FORMAT_EXPORT RunInfoTableWriter { public: - RunInfoTableWriter(std::shared_ptr&& writer, - std::shared_ptr&& schema, - std::shared_ptr const& field_locations, - std::size_t table_batch_size, - arrow::MemoryPool* pool); - RunInfoTableWriter(RunInfoTableWriter&&); - RunInfoTableWriter& operator=(RunInfoTableWriter&&); - RunInfoTableWriter(RunInfoTableWriter const&) = delete; - RunInfoTableWriter& operator=(RunInfoTableWriter const&) = delete; + RunInfoTableWriter( + std::shared_ptr && writer, + std::shared_ptr && schema, + std::shared_ptr const & field_locations, + std::size_t table_batch_size, + arrow::MemoryPool * pool); + RunInfoTableWriter(RunInfoTableWriter &&); + RunInfoTableWriter & operator=(RunInfoTableWriter &&); + RunInfoTableWriter(RunInfoTableWriter const &) = delete; + RunInfoTableWriter & operator=(RunInfoTableWriter const &) = delete; ~RunInfoTableWriter(); /// \brief Add a run info to the table, adding to the current batch. /// \param run_info_data The run info data to add. /// \returns The row index of the inserted read, or a status on failure. - Result add_run_info(RunInfoData const& run_info_data); + Result add_run_info(RunInfoData const & run_info_data); /// \brief Close this writer, signaling no further data will be written to the writer. Status close(); @@ -70,9 +73,9 @@ class POD5_FORMAT_EXPORT RunInfoTableWriter { /// \param pool Pool to be used for building table in memory. /// \returns The writer for the new table. POD5_FORMAT_EXPORT Result make_run_info_table_writer( - std::shared_ptr const& sink, - std::shared_ptr const& metadata, - std::size_t table_batch_size, - arrow::MemoryPool* pool); + std::shared_ptr const & sink, + std::shared_ptr const & metadata, + std::size_t table_batch_size, + arrow::MemoryPool * pool); } // namespace pod5 diff --git a/c++/pod5_format/schema_field_builder.h b/c++/pod5_format/schema_field_builder.h index 0277497..a93e895 100644 --- a/c++/pod5_format/schema_field_builder.h +++ b/c++/pod5_format/schema_field_builder.h @@ -20,91 +20,128 @@ class ListBuilderHelper; template <> class BuilderHelper : public arrow::FixedSizeBinaryBuilder { public: - BuilderHelper(std::shared_ptr const& uuid_type, arrow::MemoryPool* pool) - : arrow::FixedSizeBinaryBuilder(find_storage_type(uuid_type), pool) { + BuilderHelper(std::shared_ptr const & uuid_type, arrow::MemoryPool * pool) + : arrow::FixedSizeBinaryBuilder(find_storage_type(uuid_type), pool) + { assert(byte_width() == 16); } static std::shared_ptr find_storage_type( - std::shared_ptr const& uuid_type) { + std::shared_ptr const & uuid_type) + { assert(uuid_type->id() == arrow::Type::EXTENSION); auto uuid_extension = std::static_pointer_cast(uuid_type); return uuid_extension->storage_type(); } - arrow::Status Append(boost::uuids::uuid const& uuid) { - return static_cast(this)->Append(uuid.begin()); + arrow::Status Append(boost::uuids::uuid const & uuid) + { + return static_cast(this)->Append(uuid.begin()); } }; + template <> class BuilderHelper : public arrow::FloatBuilder { public: - BuilderHelper(std::shared_ptr const&, arrow::MemoryPool* pool) - : arrow::FloatBuilder(pool) {} + BuilderHelper(std::shared_ptr const &, arrow::MemoryPool * pool) + : arrow::FloatBuilder(pool) + { + } }; + template <> class BuilderHelper : public arrow::UInt8Builder { public: - BuilderHelper(std::shared_ptr const&, arrow::MemoryPool* pool) - : arrow::UInt8Builder(pool) {} + BuilderHelper(std::shared_ptr const &, arrow::MemoryPool * pool) + : arrow::UInt8Builder(pool) + { + } }; + template <> class BuilderHelper : public arrow::UInt16Builder { public: - BuilderHelper(std::shared_ptr const&, arrow::MemoryPool* pool) - : arrow::UInt16Builder(pool) {} + BuilderHelper(std::shared_ptr const &, arrow::MemoryPool * pool) + : arrow::UInt16Builder(pool) + { + } }; + template <> class BuilderHelper : public arrow::Int16Builder { public: - BuilderHelper(std::shared_ptr const&, arrow::MemoryPool* pool) - : arrow::Int16Builder(pool) {} + BuilderHelper(std::shared_ptr const &, arrow::MemoryPool * pool) + : arrow::Int16Builder(pool) + { + } }; + template <> class BuilderHelper : public arrow::UInt32Builder { public: - BuilderHelper(std::shared_ptr const&, arrow::MemoryPool* pool) - : arrow::UInt32Builder(pool) {} + BuilderHelper(std::shared_ptr const &, arrow::MemoryPool * pool) + : arrow::UInt32Builder(pool) + { + } }; + template <> class BuilderHelper : public arrow::UInt64Builder { public: - BuilderHelper(std::shared_ptr const&, arrow::MemoryPool* pool) - : arrow::UInt64Builder(pool) {} + BuilderHelper(std::shared_ptr const &, arrow::MemoryPool * pool) + : arrow::UInt64Builder(pool) + { + } }; + template <> class BuilderHelper : public arrow::BooleanBuilder { public: - BuilderHelper(std::shared_ptr const&, arrow::MemoryPool* pool) - : arrow::BooleanBuilder(pool) {} + BuilderHelper(std::shared_ptr const &, arrow::MemoryPool * pool) + : arrow::BooleanBuilder(pool) + { + } }; + template <> class BuilderHelper> : public arrow::TimestampBuilder { public: - BuilderHelper(std::shared_ptr const& type, arrow::MemoryPool* pool) - : arrow::TimestampBuilder(type, pool) {} + BuilderHelper(std::shared_ptr const & type, arrow::MemoryPool * pool) + : arrow::TimestampBuilder(type, pool) + { + } }; + template <> class BuilderHelper : public arrow::StringBuilder { public: - BuilderHelper(std::shared_ptr const&, arrow::MemoryPool* pool) - : arrow::StringBuilder(pool) {} + BuilderHelper(std::shared_ptr const &, arrow::MemoryPool * pool) + : arrow::StringBuilder(pool) + { + } }; + template <> class BuilderHelper { public: - BuilderHelper(std::shared_ptr const&, arrow::MemoryPool* pool) - : m_key_builder(std::make_shared(pool)), - m_item_builder(std::make_shared(pool)), - m_map_builder(pool, m_key_builder, m_item_builder) {} + BuilderHelper(std::shared_ptr const &, arrow::MemoryPool * pool) + : m_key_builder(std::make_shared(pool)) + , m_item_builder(std::make_shared(pool)) + , m_map_builder(pool, m_key_builder, m_item_builder) + { + } - arrow::Status Finish(std::shared_ptr* dest) { return m_map_builder.Finish(dest); } + arrow::Status Finish(std::shared_ptr * dest) + { + return m_map_builder.Finish(dest); + } arrow::Status Reserve(std::size_t rows) { return m_map_builder.Reserve(rows); } - arrow::Status Append(std::vector> const& items) { + arrow::Status Append(std::vector> const & items) + { ARROW_RETURN_NOT_OK(m_map_builder.Append()); // start new slot - for (auto const& pair : items) { + for (auto const & pair : items) { ARROW_RETURN_NOT_OK(m_key_builder->Append(pair.first)); ARROW_RETURN_NOT_OK(m_item_builder->Append(pair.second)); } @@ -120,15 +157,19 @@ class BuilderHelper { template <> class BuilderHelper : public arrow::Int16Builder { public: - BuilderHelper(std::shared_ptr const&, arrow::MemoryPool* pool) - : arrow::Int16Builder(pool) {} + BuilderHelper(std::shared_ptr const &, arrow::MemoryPool * pool) + : arrow::Int16Builder(pool) + { + } - void set_dict_writer(std::shared_ptr const& writer) { + void set_dict_writer(std::shared_ptr const & writer) + { m_dict_writer = writer; } - arrow::Status Finish(std::shared_ptr* dest) { - arrow::Int16Builder* index_builder = this; + arrow::Status Finish(std::shared_ptr * dest) + { + arrow::Int16Builder * index_builder = this; ARROW_ASSIGN_OR_RAISE(auto indices, index_builder->Finish()); ARROW_ASSIGN_OR_RAISE(*dest, m_dict_writer->build_dictionary_array(indices)); return arrow::Status::OK(); @@ -141,19 +182,23 @@ class BuilderHelper : public arrow::Int16Builder { template class ListBuilderHelper { public: - ListBuilderHelper(std::shared_ptr const&, arrow::MemoryPool* pool) - : m_array_builder(std::make_shared>(nullptr, pool)), - m_builder(pool, m_array_builder) {} + ListBuilderHelper(std::shared_ptr const &, arrow::MemoryPool * pool) + : m_array_builder(std::make_shared>(nullptr, pool)) + , m_builder(pool, m_array_builder) + { + } - arrow::Status Reserve(std::size_t rows) { + arrow::Status Reserve(std::size_t rows) + { ARROW_RETURN_NOT_OK(m_builder.Reserve(rows)); return m_array_builder->Reserve(rows); } - arrow::Status Finish(std::shared_ptr* dest) { return m_builder.Finish(dest); } + arrow::Status Finish(std::shared_ptr * dest) { return m_builder.Finish(dest); } template - arrow::Status Append(Items const& items) { + arrow::Status Append(Items const & items) + { ARROW_RETURN_NOT_OK(m_builder.Append()); // start new slot return m_array_builder->AppendValues(items.data(), items.size()); } @@ -171,22 +216,26 @@ class FieldBuilder { using BuilderTuple = std::tuple; template - FieldBuilder(std::shared_ptr const& desc_base, arrow::MemoryPool* pool) - : m_builders(typename Args::BuilderType( - desc_base->fields()[Args::WriteIndex::value]->datatype(), - pool)...) {} + FieldBuilder(std::shared_ptr const & desc_base, arrow::MemoryPool * pool) + : m_builders(typename Args::BuilderType( + desc_base->fields()[Args::WriteIndex::value]->datatype(), + pool)...) + { + } template - std::tuple_element_t& get_builder(FieldType) { + std::tuple_element_t & get_builder(FieldType) + { return std::get(m_builders); } - arrow::Result>> finish_columns() { + arrow::Result>> finish_columns() + { arrow::Status result; std::vector> columns; columns.resize(std::tuple_size::value); - detail::for_each_in_tuple(m_builders, [&](auto& element, std::size_t index) { + detail::for_each_in_tuple(m_builders, [&](auto & element, std::size_t index) { if (result.ok()) { result = element.Finish(&columns[index]); assert(columns[index] || !result.ok()); @@ -200,9 +249,10 @@ class FieldBuilder { return columns; } - arrow::Status reserve(std::size_t row_count) { + arrow::Status reserve(std::size_t row_count) + { arrow::Status result; - detail::for_each_in_tuple(m_builders, [&](auto& element, std::size_t _) { + detail::for_each_in_tuple(m_builders, [&](auto & element, std::size_t _) { if (result.ok()) { result = element.Reserve(row_count); } @@ -211,16 +261,17 @@ class FieldBuilder { } template - arrow::Status append(AppendArgs const&... args) { + arrow::Status append(AppendArgs const &... args) + { auto args_list = std::forward_as_tuple(args...); arrow::Status result; - for_each_in_tuple_zipped(m_builders, args_list, - [&](auto& builder, auto& item, std::size_t _) { - if (result.ok()) { - result = builder.Append(item); - } - }); + for_each_in_tuple_zipped( + m_builders, args_list, [&](auto & builder, auto & item, std::size_t _) { + if (result.ok()) { + result = builder.Append(item); + } + }); return result; } diff --git a/c++/pod5_format/schema_metadata.cpp b/c++/pod5_format/schema_metadata.cpp index e6980b9..e4d5af2 100644 --- a/c++/pod5_format/schema_metadata.cpp +++ b/c++/pod5_format/schema_metadata.cpp @@ -8,7 +8,8 @@ namespace pod5 { -Result parse_version_number(std::string const& ver) { +Result parse_version_number(std::string const & ver) +{ std::uint16_t components[3]; std::size_t component_index = 0; std::size_t last_char_index = 0; @@ -16,7 +17,7 @@ Result parse_version_number(std::string const& ver) { auto parse_component = [&](std::size_t last_char_index, std::size_t char_index) { auto const component_str = - std::string(ver.data() + last_char_index, ver.data() + char_index); + std::string(ver.data() + last_char_index, ver.data() + char_index); std::size_t pos = 0; int val = std::stoi(component_str, &pos); @@ -47,18 +48,21 @@ Result parse_version_number(std::string const& ver) { return Status::Invalid("Invalid component count"); } components[2] = parse_component(last_char_index, char_index); - } catch (std::exception const& e) { + } catch (std::exception const & e) { return Status::Invalid(e.what()); } return Version{components[0], components[1], components[2]}; } -Version current_build_version_number() { + +Version current_build_version_number() +{ return Version(Pod5MajorVersion, Pod5MinorVersion, Pod5RevVersion); } Result> make_schema_key_value_metadata( - SchemaMetadataDescription const& schema_metadata) { + SchemaMetadataDescription const & schema_metadata) +{ if (schema_metadata.writing_software.empty()) { return Status::Invalid("Expected writing_software to be specified for metadata"); } @@ -72,15 +76,17 @@ Result> make_schema_key_value_met } return arrow::KeyValueMetadata::Make( - {"MINKNOW:file_identifier", "MINKNOW:software", "MINKNOW:pod5_version"}, - {boost::uuids::to_string(schema_metadata.file_identifier), - schema_metadata.writing_software, schema_metadata.writing_pod5_version.to_string()}); + {"MINKNOW:file_identifier", "MINKNOW:software", "MINKNOW:pod5_version"}, + {boost::uuids::to_string(schema_metadata.file_identifier), + schema_metadata.writing_software, + schema_metadata.writing_pod5_version.to_string()}); } Result read_schema_key_value_metadata( - std::shared_ptr const& key_value_metadata) { - ARROW_ASSIGN_OR_RAISE(auto file_identifier_str, - key_value_metadata->Get("MINKNOW:file_identifier")); + std::shared_ptr const & key_value_metadata) +{ + ARROW_ASSIGN_OR_RAISE( + auto file_identifier_str, key_value_metadata->Get("MINKNOW:file_identifier")); ARROW_ASSIGN_OR_RAISE(auto software_str, key_value_metadata->Get("MINKNOW:software")); ARROW_ASSIGN_OR_RAISE(auto pod5_version_str, key_value_metadata->Get("MINKNOW:pod5_version")); ARROW_ASSIGN_OR_RAISE(auto pod5_version, parse_version_number(pod5_version_str)); @@ -88,9 +94,9 @@ Result read_schema_key_value_metadata( boost::uuids::uuid file_identifier; try { file_identifier = boost::lexical_cast(file_identifier_str); - } catch (boost::bad_lexical_cast const&) { - return Status::IOError("Schema file_identifier metadata not uuid form: '", - file_identifier_str, "'"); + } catch (boost::bad_lexical_cast const &) { + return Status::IOError( + "Schema file_identifier metadata not uuid form: '", file_identifier_str, "'"); } return SchemaMetadataDescription{file_identifier, software_str, pod5_version}; diff --git a/c++/pod5_format/schema_metadata.h b/c++/pod5_format/schema_metadata.h index a8a8d9a..8b044a2 100644 --- a/c++/pod5_format/schema_metadata.h +++ b/c++/pod5_format/schema_metadata.h @@ -13,11 +13,9 @@ namespace arrow { class KeyValueMetadata; } -namespace boost { -namespace uuids { +namespace boost { namespace uuids { struct uuid; -} -} // namespace boost +}} // namespace boost::uuids namespace pod5 { @@ -26,31 +24,38 @@ class Version { Version() : m_version(0, 0, 0) {} Version(std::uint16_t major, std::uint16_t minor, std::uint16_t revision) - : m_version(major, minor, revision) {} + : m_version(major, minor, revision) + { + } + + bool operator<(Version const & in) const { return m_version < in.m_version; } + + bool operator>(Version const & in) const { return m_version > in.m_version; } - bool operator<(Version const& in) const { return m_version < in.m_version; } - bool operator>(Version const& in) const { return m_version > in.m_version; } - bool operator==(Version const& in) const { return m_version == in.m_version; } - bool operator!=(Version const& in) const { return m_version != in.m_version; } + bool operator==(Version const & in) const { return m_version == in.m_version; } - std::string to_string() const { - return std::to_string(std::get<0>(m_version)) + "." + - std::to_string(std::get<1>(m_version)) + "." + - std::to_string(std::get<2>(m_version)); + bool operator!=(Version const & in) const { return m_version != in.m_version; } + + std::string to_string() const + { + return std::to_string(std::get<0>(m_version)) + "." + std::to_string(std::get<1>(m_version)) + + "." + std::to_string(std::get<2>(m_version)); } std::uint16_t major_version() const { return std::get<0>(m_version); } + std::uint16_t minor_version() const { return std::get<1>(m_version); } + std::uint16_t revision_version() const { return std::get<2>(m_version); } private: std::tuple parse_version_number( - std::string const& ver); + std::string const & ver); std::tuple m_version; }; -Result parse_version_number(std::string const& ver); +Result parse_version_number(std::string const & ver); Version current_build_version_number(); struct SchemaMetadataDescription { @@ -60,9 +65,9 @@ struct SchemaMetadataDescription { }; POD5_FORMAT_EXPORT Result> -make_schema_key_value_metadata(SchemaMetadataDescription const& schema_metadata); +make_schema_key_value_metadata(SchemaMetadataDescription const & schema_metadata); POD5_FORMAT_EXPORT Result read_schema_key_value_metadata( - std::shared_ptr const& key_value_metadata); + std::shared_ptr const & key_value_metadata); } // namespace pod5 diff --git a/c++/pod5_format/schema_utils.cpp b/c++/pod5_format/schema_utils.cpp index c2e643b..a364e85 100644 --- a/c++/pod5_format/schema_utils.cpp +++ b/c++/pod5_format/schema_utils.cpp @@ -6,10 +6,11 @@ namespace pod5 { /// \param metadata Metadata to be applied to the schema. /// \returns The schema for a read table. std::shared_ptr SchemaDescriptionBase::make_writer_schema( - std::shared_ptr const& metadata) const { + std::shared_ptr const & metadata) const +{ auto const latest_version = latest_table_version(); arrow::FieldVector writer_fields; - for (auto& field : fields()) { + for (auto & field : fields()) { if (field->removed_table_spec_version() > latest_version) { writer_fields.emplace_back(arrow::field(field->name(), field->datatype())); } @@ -17,30 +18,35 @@ std::shared_ptr SchemaDescriptionBase::make_writer_schema( return arrow::schema(writer_fields, metadata); } -Status SchemaDescriptionBase::read_schema(std::shared_ptr dest_schema, - SchemaMetadataDescription const& schema_metadata, - std::shared_ptr const& schema) { +Status SchemaDescriptionBase::read_schema( + std::shared_ptr dest_schema, + SchemaMetadataDescription const & schema_metadata, + std::shared_ptr const & schema) +{ dest_schema->m_table_spec_version = - dest_schema->table_version_from_file_version(schema_metadata.writing_pod5_version); + dest_schema->table_version_from_file_version(schema_metadata.writing_pod5_version); - for (auto& field : dest_schema->fields()) { - if (dest_schema->table_version() < field->added_table_spec_version() || - dest_schema->table_version() >= field->removed_table_spec_version()) { + for (auto & field : dest_schema->fields()) { + if (dest_schema->table_version() < field->added_table_spec_version() + || dest_schema->table_version() >= field->removed_table_spec_version()) + { continue; } - auto const& datatype = field->datatype(); + auto const & datatype = field->datatype(); int field_index = 0; if (datatype->id() == arrow::Type::DICTIONARY) { auto dict_type = std::static_pointer_cast(datatype); if (dict_type->value_type()->id() == arrow::Type::STRUCT) { std::shared_ptr value_type; - ARROW_ASSIGN_OR_RAISE(field_index, find_dict_field(schema, field->name().c_str(), - arrow::int16(), &value_type)); + ARROW_ASSIGN_OR_RAISE( + field_index, + find_dict_field(schema, field->name().c_str(), arrow::int16(), &value_type)); } else { std::shared_ptr value_type; - ARROW_ASSIGN_OR_RAISE(field_index, find_dict_field(schema, field->name().c_str(), - arrow::int16(), &value_type)); + ARROW_ASSIGN_OR_RAISE( + field_index, + find_dict_field(schema, field->name().c_str(), arrow::int16(), &value_type)); } } else { ARROW_ASSIGN_OR_RAISE(field_index, find_field(schema, field->name().c_str(), datatype)); diff --git a/c++/pod5_format/schema_utils.h b/c++/pod5_format/schema_utils.h index ccfcb9a..d3fad05 100644 --- a/c++/pod5_format/schema_utils.h +++ b/c++/pod5_format/schema_utils.h @@ -7,8 +7,10 @@ namespace pod5 { -inline arrow::Result find_field_untyped(std::shared_ptr const& schema, - char const* name) { +inline arrow::Result find_field_untyped( + std::shared_ptr const & schema, + char const * name) +{ auto const field_idx = schema->GetFieldIndex(name); if (field_idx == -1) { return Status::TypeError("Schema missing field '", name, "'"); @@ -17,27 +19,31 @@ inline arrow::Result find_field_untyped(std::shared_ptr cons return field_idx; } -inline arrow::Result find_field(std::shared_ptr const& schema, - char const* name, - std::shared_ptr const& expected_data_type) { +inline arrow::Result find_field( + std::shared_ptr const & schema, + char const * name, + std::shared_ptr const & expected_data_type) +{ ARROW_ASSIGN_OR_RAISE(auto field_idx, find_field_untyped(schema, name)); auto const field = schema->field(field_idx); auto const type = field->type(); if (!type->Equals(expected_data_type)) { - return Status::TypeError("Schema field '", name, "' is incorrect type: '", type->name(), - "'"); + return Status::TypeError( + "Schema field '", name, "' is incorrect type: '", type->name(), "'"); } return field_idx; } template -inline arrow::Result find_dict_field(std::shared_ptr const& schema, - char const* name, - std::shared_ptr const& index_type, - std::shared_ptr* value_type) { +inline arrow::Result find_dict_field( + std::shared_ptr const & schema, + char const * name, + std::shared_ptr const & index_type, + std::shared_ptr * value_type) +{ ARROW_ASSIGN_OR_RAISE(auto field_idx, find_field_untyped(schema, name)); auto const field = schema->field(field_idx); @@ -47,22 +53,23 @@ inline arrow::Result find_dict_field(std::shared_ptr const& } if (!type->index_type()->Equals(index_type)) { - return Status::TypeError("Schema field '", name, "' is incorrect type: '", type->name(), - "'"); + return Status::TypeError( + "Schema field '", name, "' is incorrect type: '", type->name(), "'"); } *value_type = std::dynamic_pointer_cast(type->value_type()); if (!*value_type) { - return Status::TypeError("Dictionary value was unexpected type: ", - type->value_type()->name()); + return Status::TypeError( + "Dictionary value was unexpected type: ", type->value_type()->name()); } return field_idx; } template std::shared_ptr find_column( - std::shared_ptr const& batch, - FieldType const& field) { + std::shared_ptr const & batch, + FieldType const & field) +{ auto const field_base = batch->column(field.field_index()); return std::static_pointer_cast(field_base); } @@ -87,13 +94,13 @@ class TableSpecVersion { UnderlyingType as_int() const { return m_version; } - bool operator<(TableSpecVersion const& other) const { return m_version < other.m_version; } + bool operator<(TableSpecVersion const & other) const { return m_version < other.m_version; } - bool operator>(TableSpecVersion const& other) const { return m_version > other.m_version; } + bool operator>(TableSpecVersion const & other) const { return m_version > other.m_version; } - bool operator<=(TableSpecVersion const& other) const { return m_version <= other.m_version; } + bool operator<=(TableSpecVersion const & other) const { return m_version <= other.m_version; } - bool operator>=(TableSpecVersion const& other) const { return m_version >= other.m_version; } + bool operator>=(TableSpecVersion const & other) const { return m_version >= other.m_version; } private: TableSpecVersion(UnderlyingType version) : m_version(version) {} @@ -104,34 +111,40 @@ class TableSpecVersion { class SchemaDescriptionBase { public: SchemaDescriptionBase(TableSpecVersion version) : m_table_spec_version(version) {} + virtual ~SchemaDescriptionBase() = default; - void add_field(FieldBase* field) { m_fields.push_back(field); } + void add_field(FieldBase * field) { m_fields.push_back(field); } + + std::vector const & fields() { return m_fields; } - std::vector const& fields() { return m_fields; } - std::vector const& fields() const { - return reinterpret_cast const&>(m_fields); + std::vector const & fields() const + { + return reinterpret_cast const &>(m_fields); } - TableSpecVersion latest_table_version() const { + TableSpecVersion latest_table_version() const + { return table_version_from_file_version(current_build_version_number()); } virtual TableSpecVersion table_version_from_file_version(Version file_version) const = 0; + TableSpecVersion table_version() const { return m_table_spec_version; } /// \brief Make a new schema for a read table to be written (will only contain fields which are written in the latest version). /// \param metadata Metadata to be applied to the schema. /// \returns The schema for a read table. std::shared_ptr make_writer_schema( - std::shared_ptr const& metadata) const; + std::shared_ptr const & metadata) const; - static Status read_schema(std::shared_ptr dest_schema, - SchemaMetadataDescription const& schema_metadata, - std::shared_ptr const& schema); + static Status read_schema( + std::shared_ptr dest_schema, + SchemaMetadataDescription const & schema_metadata, + std::shared_ptr const & schema); private: - std::vector m_fields; + std::vector m_fields; TableSpecVersion m_table_spec_version; }; @@ -144,24 +157,30 @@ class ListBuilderHelper; class FieldBase { public: - FieldBase(SchemaDescriptionBase* owner, - int field_index, - std::string name, - std::shared_ptr const& datatype, - TableSpecVersion added_table_spec_version = TableSpecVersion::first_version(), - TableSpecVersion removed_table_spec_version = TableSpecVersion::unknown_version()) - : m_name(name), - m_datatype(datatype), - m_field_index(field_index), - m_added_table_spec_version(added_table_spec_version), - m_removed_table_spec_version(removed_table_spec_version) { + FieldBase( + SchemaDescriptionBase * owner, + int field_index, + std::string name, + std::shared_ptr const & datatype, + TableSpecVersion added_table_spec_version = TableSpecVersion::first_version(), + TableSpecVersion removed_table_spec_version = TableSpecVersion::unknown_version()) + : m_name(name) + , m_datatype(datatype) + , m_field_index(field_index) + , m_added_table_spec_version(added_table_spec_version) + , m_removed_table_spec_version(removed_table_spec_version) + { owner->add_field(this); } - std::string const& name() const { return m_name; } - std::shared_ptr const& datatype() const { return m_datatype; } + std::string const & name() const { return m_name; } + + std::shared_ptr const & datatype() const { return m_datatype; } + int field_index() const { return m_field_index; } + TableSpecVersion added_table_spec_version() const { return m_added_table_spec_version; } + TableSpecVersion removed_table_spec_version() const { return m_removed_table_spec_version; } void set_field_index(int index) { m_field_index = index; } @@ -182,17 +201,21 @@ struct Field : public FieldBase { using ArrayType = ArrayType_; using BuilderType = detail::BuilderHelper; - Field(SchemaDescriptionBase* owner, - std::string name, - std::shared_ptr const& datatype, - TableSpecVersion added_table_spec_version = TableSpecVersion::first_version(), - TableSpecVersion removed_table_spec_version = TableSpecVersion::unknown_version()) - : FieldBase(owner, - WriteIndex::value, - name, - datatype, - added_table_spec_version, - removed_table_spec_version) {} + Field( + SchemaDescriptionBase * owner, + std::string name, + std::shared_ptr const & datatype, + TableSpecVersion added_table_spec_version = TableSpecVersion::first_version(), + TableSpecVersion removed_table_spec_version = TableSpecVersion::unknown_version()) + : FieldBase( + owner, + WriteIndex::value, + name, + datatype, + added_table_spec_version, + removed_table_spec_version) + { + } }; template @@ -200,16 +223,20 @@ struct ListField : public Field { using ElementType = ElementType_; using BuilderType = detail::ListBuilderHelper; - ListField(SchemaDescriptionBase* owner, - std::string name, - std::shared_ptr const& datatype, - TableSpecVersion added_table_spec_version = TableSpecVersion::first_version(), - TableSpecVersion removed_table_spec_version = TableSpecVersion::unknown_version()) - : Field(owner, - name, - datatype, - added_table_spec_version, - removed_table_spec_version) {} + ListField( + SchemaDescriptionBase * owner, + std::string name, + std::shared_ptr const & datatype, + TableSpecVersion added_table_spec_version = TableSpecVersion::first_version(), + TableSpecVersion removed_table_spec_version = TableSpecVersion::unknown_version()) + : Field( + owner, + name, + datatype, + added_table_spec_version, + removed_table_spec_version) + { + } }; template diff --git a/c++/pod5_format/signal_compression.cpp b/c++/pod5_format/signal_compression.cpp index 60ac514..a22a977 100644 --- a/c++/pod5_format/signal_compression.cpp +++ b/c++/pod5_format/signal_compression.cpp @@ -8,15 +8,18 @@ namespace pod5 { -std::size_t compressed_signal_max_size(std::size_t sample_count) { +std::size_t compressed_signal_max_size(std::size_t sample_count) +{ auto const max_svb_size = svb16_max_encoded_length(sample_count); auto const zstd_compressed_max_size = ZSTD_compressBound(max_svb_size); return zstd_compressed_max_size; } -arrow::Result compress_signal(gsl::span const& samples, - arrow::MemoryPool* pool, - gsl::span const& destination) { +arrow::Result compress_signal( + gsl::span const & samples, + arrow::MemoryPool * pool, + gsl::span const & destination) +{ // First compress the data using svb: auto const max_size = svb16_max_encoded_length(samples.size()); ARROW_ASSIGN_OR_RAISE(auto intermediate, arrow::AllocateResizableBuffer(max_size, pool)); @@ -24,7 +27,7 @@ arrow::Result compress_signal(gsl::span const& sa static constexpr bool UseDelta = true; static constexpr bool UseZigzag = true; auto const encoded_count = svb16::encode( - samples.data(), intermediate->mutable_data(), samples.size()); + samples.data(), intermediate->mutable_data(), samples.size()); ARROW_RETURN_NOT_OK(intermediate->Resize(encoded_count)); // Now compress the svb data using zstd: @@ -37,8 +40,8 @@ arrow::Result compress_signal(gsl::span const& sa * If you are doing many compressions, you may want to reuse the context. * See the multiple_simple_compression.c example. */ - size_t const compressed_size = ZSTD_compress(destination.data(), destination.size(), - intermediate->data(), intermediate->size(), 1); + size_t const compressed_size = ZSTD_compress( + destination.data(), destination.size(), intermediate->data(), intermediate->size(), 1); if (ZSTD_isError(compressed_size)) { return pod5::Status::Invalid("Failed to compress data"); } @@ -46,36 +49,40 @@ arrow::Result compress_signal(gsl::span const& sa } arrow::Result> compress_signal( - gsl::span const& samples, - arrow::MemoryPool* pool) { + gsl::span const & samples, + arrow::MemoryPool * pool) +{ ARROW_ASSIGN_OR_RAISE( - std::shared_ptr out, - arrow::AllocateResizableBuffer(compressed_signal_max_size(samples.size()), pool)); + std::shared_ptr out, + arrow::AllocateResizableBuffer(compressed_signal_max_size(samples.size()), pool)); ARROW_ASSIGN_OR_RAISE( - auto final_size, - compress_signal(samples, pool, gsl::make_span(out->mutable_data(), out->size()))); + auto final_size, + compress_signal(samples, pool, gsl::make_span(out->mutable_data(), out->size()))); ARROW_RETURN_NOT_OK(out->Resize(final_size)); return out; } POD5_FORMAT_EXPORT arrow::Status decompress_signal( - gsl::span const& compressed_bytes, - arrow::MemoryPool* pool, - gsl::span const& destination) { + gsl::span const & compressed_bytes, + arrow::MemoryPool * pool, + gsl::span const & destination) +{ // First decompress the data using zstd: unsigned long long const decompressed_zstd_size = - ZSTD_getFrameContentSize(compressed_bytes.data(), compressed_bytes.size()); + ZSTD_getFrameContentSize(compressed_bytes.data(), compressed_bytes.size()); if (ZSTD_isError(decompressed_zstd_size)) { return pod5::Status::Invalid("Input data not compressed by zstd"); } - ARROW_ASSIGN_OR_RAISE(auto intermediate, - arrow::AllocateResizableBuffer(decompressed_zstd_size, pool)); - size_t const decompress_res = - ZSTD_decompress(intermediate->mutable_data(), intermediate->size(), - compressed_bytes.data(), compressed_bytes.size()); + ARROW_ASSIGN_OR_RAISE( + auto intermediate, arrow::AllocateResizableBuffer(decompressed_zstd_size, pool)); + size_t const decompress_res = ZSTD_decompress( + intermediate->mutable_data(), + intermediate->size(), + compressed_bytes.data(), + compressed_bytes.size()); if (ZSTD_isError(decompress_res)) { return pod5::Status::Invalid("Input data failed to compress using zstd"); } @@ -84,8 +91,9 @@ POD5_FORMAT_EXPORT arrow::Status decompress_signal( static constexpr bool UseDelta = true; static constexpr bool UseZigzag = true; auto consumed_count = svb16::decode( - reinterpret_cast(destination.data()), intermediate->data(), - destination.size()); + reinterpret_cast(destination.data()), + intermediate->data(), + destination.size()); if (consumed_count != (std::size_t)intermediate->size()) { return pod5::Status::Invalid("Remaining data at end of signal buffer"); @@ -95,11 +103,13 @@ POD5_FORMAT_EXPORT arrow::Status decompress_signal( } arrow::Result> decompress_signal( - gsl::span const& compressed_bytes, - std::uint32_t samples_count, - arrow::MemoryPool* pool) { - ARROW_ASSIGN_OR_RAISE(std::shared_ptr out, - arrow::AllocateResizableBuffer(samples_count * sizeof(SampleType), pool)); + gsl::span const & compressed_bytes, + std::uint32_t samples_count, + arrow::MemoryPool * pool) +{ + ARROW_ASSIGN_OR_RAISE( + std::shared_ptr out, + arrow::AllocateResizableBuffer(samples_count * sizeof(SampleType), pool)); auto signal_span = gsl::make_span(out->mutable_data(), out->size()).as_span(); diff --git a/c++/pod5_format/signal_compression.h b/c++/pod5_format/signal_compression.h index ec02656..affbb6d 100644 --- a/c++/pod5_format/signal_compression.h +++ b/c++/pod5_format/signal_compression.h @@ -17,22 +17,22 @@ using SampleType = std::int16_t; POD5_FORMAT_EXPORT std::size_t compressed_signal_max_size(std::size_t sample_count); POD5_FORMAT_EXPORT arrow::Result compress_signal( - gsl::span const& samples, - arrow::MemoryPool* pool, - gsl::span const& destination); + gsl::span const & samples, + arrow::MemoryPool * pool, + gsl::span const & destination); POD5_FORMAT_EXPORT arrow::Result> compress_signal( - gsl::span const& samples, - arrow::MemoryPool* pool); + gsl::span const & samples, + arrow::MemoryPool * pool); POD5_FORMAT_EXPORT arrow::Result> decompress_signal( - gsl::span const& compressed_bytes, - std::uint32_t samples_count, - arrow::MemoryPool* pool); + gsl::span const & compressed_bytes, + std::uint32_t samples_count, + arrow::MemoryPool * pool); POD5_FORMAT_EXPORT arrow::Status decompress_signal( - gsl::span const& compressed_bytes, - arrow::MemoryPool* pool, - gsl::span const& destination); + gsl::span const & compressed_bytes, + arrow::MemoryPool * pool, + gsl::span const & destination); } // namespace pod5 diff --git a/c++/pod5_format/signal_table_reader.cpp b/c++/pod5_format/signal_table_reader.cpp index ad03211..404f8fe 100644 --- a/c++/pod5_format/signal_table_reader.cpp +++ b/c++/pod5_format/signal_table_reader.cpp @@ -11,29 +11,39 @@ namespace pod5 { -SignalTableRecordBatch::SignalTableRecordBatch(std::shared_ptr&& batch, - SignalTableSchemaDescription field_locations, - arrow::MemoryPool* pool) - : TableRecordBatch(std::move(batch)), m_field_locations(field_locations), m_pool(pool) {} +SignalTableRecordBatch::SignalTableRecordBatch( + std::shared_ptr && batch, + SignalTableSchemaDescription field_locations, + arrow::MemoryPool * pool) +: TableRecordBatch(std::move(batch)) +, m_field_locations(field_locations) +, m_pool(pool) +{ +} -std::shared_ptr SignalTableRecordBatch::read_id_column() const { +std::shared_ptr SignalTableRecordBatch::read_id_column() const +{ return std::static_pointer_cast(batch()->column(m_field_locations.read_id)); } -std::shared_ptr SignalTableRecordBatch::uncompressed_signal_column() const { +std::shared_ptr SignalTableRecordBatch::uncompressed_signal_column() const +{ return std::static_pointer_cast( - batch()->column(m_field_locations.signal)); + batch()->column(m_field_locations.signal)); } -std::shared_ptr SignalTableRecordBatch::vbz_signal_column() const { +std::shared_ptr SignalTableRecordBatch::vbz_signal_column() const +{ return std::static_pointer_cast(batch()->column(m_field_locations.signal)); } -std::shared_ptr SignalTableRecordBatch::samples_column() const { +std::shared_ptr SignalTableRecordBatch::samples_column() const +{ return std::static_pointer_cast(batch()->column(m_field_locations.samples)); } -Result SignalTableRecordBatch::samples_byte_count(std::size_t row_index) const { +Result SignalTableRecordBatch::samples_byte_count(std::size_t row_index) const +{ switch (m_field_locations.signal_type) { case SignalType::UncompressedSignal: { auto signal_column = uncompressed_signal_column(); @@ -50,25 +60,31 @@ Result SignalTableRecordBatch::samples_byte_count(std::size_t row_i return pod5::Status::Invalid("Unknown signal type"); } -Status SignalTableRecordBatch::extract_signal_row(std::size_t row_index, - gsl::span samples) const { +Status SignalTableRecordBatch::extract_signal_row( + std::size_t row_index, + gsl::span samples) const +{ if (row_index >= num_rows()) { - return pod5::Status::Invalid("Queried signal row ", row_index, - " is outside the available rows (", num_rows(), " in batch)"); + return pod5::Status::Invalid( + "Queried signal row ", + row_index, + " is outside the available rows (", + num_rows(), + " in batch)"); } auto sample_count = samples_column(); auto samples_in_row = sample_count->Value(row_index); if (samples_in_row != samples.size()) { - return pod5::Status::Invalid("Unexpected size for sample array ", samples.size(), - " expected ", samples_in_row); + return pod5::Status::Invalid( + "Unexpected size for sample array ", samples.size(), " expected ", samples_in_row); } switch (m_field_locations.signal_type) { case SignalType::UncompressedSignal: { auto signal_column = uncompressed_signal_column(); auto signal = - std::static_pointer_cast(signal_column->value_slice(row_index)); + std::static_pointer_cast(signal_column->value_slice(row_index)); std::copy(signal->raw_values(), signal->raw_values() + signal->length(), samples.begin()); return Status::OK(); } @@ -83,20 +99,25 @@ Status SignalTableRecordBatch::extract_signal_row(std::size_t row_index, } Result> SignalTableRecordBatch::extract_signal_row_inplace( - std::size_t row_index) const { + std::size_t row_index) const +{ if (row_index >= num_rows()) { - return pod5::Status::Invalid("Queried signal row ", row_index, - " is outside the available rows (", num_rows(), " in batch)"); + return pod5::Status::Invalid( + "Queried signal row ", + row_index, + " is outside the available rows (", + num_rows(), + " in batch)"); } switch (m_field_locations.signal_type) { case SignalType::UncompressedSignal: { auto signal_column = uncompressed_signal_column(); auto const value_slice = - std::static_pointer_cast(signal_column->value_slice(row_index)); + std::static_pointer_cast(signal_column->value_slice(row_index)); auto const element_size = - sizeof(std::remove_reference::type::TypeClass); + sizeof(std::remove_reference::type::TypeClass); auto const values = value_slice->values(); auto offset = signal_column->value_offset(row_index); @@ -114,36 +135,43 @@ Result> SignalTableRecordBatch::extract_signal_ro //--------------------------------------------------------------------------------------------------------------------- -SignalTableReader::SignalTableReader(std::shared_ptr&& input_source, - std::shared_ptr&& reader, - SignalTableSchemaDescription field_locations, - SchemaMetadataDescription&& schema_metadata, - std::size_t num_record_batches, - std::size_t batch_size, - arrow::MemoryPool* pool) - : TableReader(std::move(input_source), std::move(reader), std::move(schema_metadata), pool), - m_field_locations(field_locations), - m_pool(pool), - m_table_batches(num_record_batches), - m_batch_size(batch_size) {} - -SignalTableReader::SignalTableReader(SignalTableReader&& other) - : TableReader(std::move(other)), - m_field_locations(std::move(other.m_field_locations)), - m_pool(other.m_pool), - m_table_batches(std::move(other.m_table_batches)), - m_batch_size(other.m_batch_size) {} - -SignalTableReader& SignalTableReader::operator=(SignalTableReader&& other) { +SignalTableReader::SignalTableReader( + std::shared_ptr && input_source, + std::shared_ptr && reader, + SignalTableSchemaDescription field_locations, + SchemaMetadataDescription && schema_metadata, + std::size_t num_record_batches, + std::size_t batch_size, + arrow::MemoryPool * pool) +: TableReader(std::move(input_source), std::move(reader), std::move(schema_metadata), pool) +, m_field_locations(field_locations) +, m_pool(pool) +, m_table_batches(num_record_batches) +, m_batch_size(batch_size) +{ +} + +SignalTableReader::SignalTableReader(SignalTableReader && other) +: TableReader(std::move(other)) +, m_field_locations(std::move(other.m_field_locations)) +, m_pool(other.m_pool) +, m_table_batches(std::move(other.m_table_batches)) +, m_batch_size(other.m_batch_size) +{ +} + +SignalTableReader & SignalTableReader::operator=(SignalTableReader && other) +{ m_field_locations = std::move(other.m_field_locations); m_pool = other.m_pool; m_batch_size = other.m_batch_size; m_table_batches = std::move(other.m_table_batches); - static_cast(*this) = std::move(static_cast(other)); + static_cast(*this) = std::move(static_cast(other)); return *this; } -Result SignalTableReader::read_record_batch(std::size_t i) const { +Result SignalTableReader::read_record_batch(std::size_t i) const +{ if (m_table_batches[i]) { return *m_table_batches[i]; } @@ -158,8 +186,10 @@ Result SignalTableReader::read_record_batch(std::size_t return *m_table_batches[i]; } -Result SignalTableReader::signal_batch_for_row_id(std::uint64_t row, - std::size_t* batch_row) const { +Result SignalTableReader::signal_batch_for_row_id( + std::uint64_t row, + std::size_t * batch_row) const +{ auto batch = row / m_batch_size; if (batch_row) { @@ -174,31 +204,34 @@ Result SignalTableReader::signal_batch_for_row_id(std::uint64_t row } Result SignalTableReader::extract_sample_count( - gsl::span const& row_indices) const { + gsl::span const & row_indices) const +{ std::size_t sample_count = 0; - for (auto const& signal_row : row_indices) { + for (auto const & signal_row : row_indices) { std::size_t batch_row = 0; - ARROW_ASSIGN_OR_RAISE(auto const signal_batch_index, - signal_batch_for_row_id(signal_row, &batch_row)); + ARROW_ASSIGN_OR_RAISE( + auto const signal_batch_index, signal_batch_for_row_id(signal_row, &batch_row)); - ARROW_ASSIGN_OR_RAISE(auto const& signal_batch, read_record_batch(signal_batch_index)); - auto const& samples_column = signal_batch.samples_column(); + ARROW_ASSIGN_OR_RAISE(auto const & signal_batch, read_record_batch(signal_batch_index)); + auto const & samples_column = signal_batch.samples_column(); sample_count += samples_column->Value(batch_row); } return sample_count; } -Status SignalTableReader::extract_samples(gsl::span const& row_indices, - gsl::span const& output_samples) const { +Status SignalTableReader::extract_samples( + gsl::span const & row_indices, + gsl::span const & output_samples) const +{ std::size_t sample_count = 0; - for (auto const& signal_row : row_indices) { + for (auto const & signal_row : row_indices) { std::size_t batch_row = 0; - ARROW_ASSIGN_OR_RAISE(auto const signal_batch_index, - signal_batch_for_row_id(signal_row, &batch_row)); + ARROW_ASSIGN_OR_RAISE( + auto const signal_batch_index, signal_batch_for_row_id(signal_row, &batch_row)); - ARROW_ASSIGN_OR_RAISE(auto const& signal_batch, read_record_batch(signal_batch_index)); - auto const& samples_column = signal_batch.samples_column(); + ARROW_ASSIGN_OR_RAISE(auto const & signal_batch, read_record_batch(signal_batch_index)); + auto const & samples_column = signal_batch.samples_column(); auto const row_samples_count = samples_column->Value(batch_row); std::size_t const sample_start = sample_count; sample_count += row_samples_count; @@ -207,27 +240,28 @@ Status SignalTableReader::extract_samples(gsl::span const& } ARROW_RETURN_NOT_OK(signal_batch.extract_signal_row( - batch_row, output_samples.subspan(sample_start, row_samples_count))); + batch_row, output_samples.subspan(sample_start, row_samples_count))); } return Status::OK(); } Result>> SignalTableReader::extract_samples_inplace( - gsl::span const& row_indices, - std::vector& sample_count) const { + gsl::span const & row_indices, + std::vector & sample_count) const +{ std::vector> sample_buffers; - for (auto const& signal_row : row_indices) { + for (auto const & signal_row : row_indices) { std::size_t batch_row = 0; - ARROW_ASSIGN_OR_RAISE(auto const signal_batch_index, - signal_batch_for_row_id(signal_row, &batch_row)); + ARROW_ASSIGN_OR_RAISE( + auto const signal_batch_index, signal_batch_for_row_id(signal_row, &batch_row)); - ARROW_ASSIGN_OR_RAISE(auto const& signal_batch, read_record_batch(signal_batch_index)); + ARROW_ASSIGN_OR_RAISE(auto const & signal_batch, read_record_batch(signal_batch_index)); ARROW_ASSIGN_OR_RAISE(auto signal_data, signal_batch.extract_signal_row_inplace(batch_row)); sample_buffers.emplace_back(std::move(signal_data)); - auto const& samples_column = signal_batch.samples_column(); + auto const & samples_column = signal_batch.samples_column(); sample_count.push_back(samples_column->Value(batch_row)); } return sample_buffers; @@ -237,8 +271,9 @@ SignalType SignalTableReader::signal_type() const { return m_field_locations.sig //--------------------------------------------------------------------------------------------------------------------- Result make_signal_table_reader( - std::shared_ptr const& input, - arrow::MemoryPool* pool) { + std::shared_ptr const & input, + arrow::MemoryPool * pool) +{ arrow::ipc::IpcReadOptions options; options.memory_pool = pool; @@ -248,8 +283,8 @@ Result make_signal_table_reader( if (!read_metadata_key_values) { return Status::IOError("Missing metadata on signal table schema"); } - ARROW_ASSIGN_OR_RAISE(auto read_metadata, - read_schema_key_value_metadata(read_metadata_key_values)); + ARROW_ASSIGN_OR_RAISE( + auto read_metadata, read_schema_key_value_metadata(read_metadata_key_values)); ARROW_ASSIGN_OR_RAISE(auto field_locations, read_signal_table_schema(reader->schema())); std::size_t const num_record_batches = reader->num_record_batches(); @@ -259,8 +294,14 @@ Result make_signal_table_reader( batch_size = batch_zero->num_rows(); } - return SignalTableReader({input}, std::move(reader), field_locations, std::move(read_metadata), - num_record_batches, batch_size, pool); + return SignalTableReader( + {input}, + std::move(reader), + field_locations, + std::move(read_metadata), + num_record_batches, + batch_size, + pool); } } // namespace pod5 diff --git a/c++/pod5_format/signal_table_reader.h b/c++/pod5_format/signal_table_reader.h index 1688646..8267d08 100644 --- a/c++/pod5_format/signal_table_reader.h +++ b/c++/pod5_format/signal_table_reader.h @@ -17,9 +17,11 @@ namespace arrow { class Schema; + namespace io { class RandomAccessFile; } + namespace ipc { class RecordBatchFileReader; } @@ -29,9 +31,10 @@ namespace pod5 { class POD5_FORMAT_EXPORT SignalTableRecordBatch : public TableRecordBatch { public: - SignalTableRecordBatch(std::shared_ptr&& batch, - SignalTableSchemaDescription field_locations, - arrow::MemoryPool* pool); + SignalTableRecordBatch( + std::shared_ptr && batch, + SignalTableSchemaDescription field_locations, + arrow::MemoryPool * pool); std::shared_ptr read_id_column() const; std::shared_ptr uncompressed_signal_column() const; @@ -46,50 +49,52 @@ class POD5_FORMAT_EXPORT SignalTableRecordBatch : public TableRecordBatch { private: SignalTableSchemaDescription m_field_locations; - arrow::MemoryPool* m_pool; + arrow::MemoryPool * m_pool; }; class POD5_FORMAT_EXPORT SignalTableReader : public TableReader { public: - SignalTableReader(std::shared_ptr&& input_source, - std::shared_ptr&& reader, - SignalTableSchemaDescription field_locations, - SchemaMetadataDescription&& schema_metadata, - std::size_t num_record_batches, - std::size_t batch_size, - arrow::MemoryPool* pool); - - SignalTableReader(SignalTableReader&&); - SignalTableReader& operator=(SignalTableReader&&); + SignalTableReader( + std::shared_ptr && input_source, + std::shared_ptr && reader, + SignalTableSchemaDescription field_locations, + SchemaMetadataDescription && schema_metadata, + std::size_t num_record_batches, + std::size_t batch_size, + arrow::MemoryPool * pool); + + SignalTableReader(SignalTableReader &&); + SignalTableReader & operator=(SignalTableReader &&); Result read_record_batch(std::size_t i) const; - Result signal_batch_for_row_id(std::uint64_t row, std::size_t* batch_row) const; + Result signal_batch_for_row_id(std::uint64_t row, std::size_t * batch_row) const; /// \brief Find the number of samples in a given list of rows. /// \param row_indices The rows to query for sample ount. /// \returns The sum of all sample counts on input rows. Result extract_sample_count( - gsl::span const& row_indices) const; + gsl::span const & row_indices) const; /// \brief Extract the samples for a list of rows. /// \param row_indices The rows to query for samples. /// \param output_samples The output samples from the rows. Data in the vector is cleared before appending. - Status extract_samples(gsl::span const& row_indices, - gsl::span const& output_samples) const; + Status extract_samples( + gsl::span const & row_indices, + gsl::span const & output_samples) const; /// \brief Extract the samples as written in the arrow table for a list of rows. /// \param row_indices The rows to query for samples. Result>> extract_samples_inplace( - gsl::span const& row_indices, - std::vector& sample_count) const; + gsl::span const & row_indices, + std::vector & sample_count) const; /// \brief Find the signal type of this writer SignalType signal_type() const; private: SignalTableSchemaDescription m_field_locations; - arrow::MemoryPool* m_pool; + arrow::MemoryPool * m_pool; mutable std::mutex m_batch_get_mutex; mutable std::vector> m_table_batches; @@ -98,7 +103,7 @@ class POD5_FORMAT_EXPORT SignalTableReader : public TableReader { }; POD5_FORMAT_EXPORT Result make_signal_table_reader( - std::shared_ptr const& sink, - arrow::MemoryPool* pool); + std::shared_ptr const & sink, + arrow::MemoryPool * pool); } // namespace pod5 diff --git a/c++/pod5_format/signal_table_schema.cpp b/c++/pod5_format/signal_table_schema.cpp index 906ed48..70a0563 100644 --- a/c++/pod5_format/signal_table_schema.cpp +++ b/c++/pod5_format/signal_table_schema.cpp @@ -8,9 +8,10 @@ namespace pod5 { std::shared_ptr make_signal_table_schema( - SignalType signal_type, - std::shared_ptr const& metadata, - SignalTableSchemaDescription* field_locations) { + SignalType signal_type, + std::shared_ptr const & metadata, + SignalTableSchemaDescription * field_locations) +{ auto const uuid_type = uuid(); if (field_locations) { @@ -29,16 +30,17 @@ std::shared_ptr make_signal_table_schema( } return arrow::schema( - { - arrow::field("read_id", uuid_type), - arrow::field("signal", signal_schema_type), - arrow::field("samples", arrow::uint32()), - }, - metadata); + { + arrow::field("read_id", uuid_type), + arrow::field("signal", signal_schema_type), + arrow::field("samples", arrow::uint32()), + }, + metadata); } Result read_signal_table_schema( - std::shared_ptr const& schema) { + std::shared_ptr const & schema) +{ ARROW_ASSIGN_OR_RAISE(auto read_id_field_idx, find_field(schema, "read_id", uuid())); ARROW_ASSIGN_OR_RAISE(auto samples_field_idx, find_field(schema, "samples", arrow::uint32())); @@ -50,20 +52,20 @@ Result read_signal_table_schema( auto const signal_arrow_type = signal_field->type(); if (signal_arrow_type->id() == arrow::Type::LARGE_LIST) { auto const signal_list_field = - std::static_pointer_cast(signal_field->type()); + std::static_pointer_cast(signal_field->type()); if (signal_list_field->value_type()->id() != arrow::Type::INT16) { return Status::TypeError("Schema field 'signal' list value type is incorrect type"); } } else if (signal_arrow_type->Equals(vbz_signal())) { signal_type = SignalType::VbzSignal; } else { - return Status::TypeError("Schema field 'signal' is incorrect type: '", - signal_arrow_type->name(), "'"); + return Status::TypeError( + "Schema field 'signal' is incorrect type: '", signal_arrow_type->name(), "'"); } } - return SignalTableSchemaDescription{signal_type, read_id_field_idx, signal_field_idx, - samples_field_idx}; + return SignalTableSchemaDescription{ + signal_type, read_id_field_idx, signal_field_idx, samples_field_idx}; } } // namespace pod5 diff --git a/c++/pod5_format/signal_table_schema.h b/c++/pod5_format/signal_table_schema.h index c2fc62c..0d8c834 100644 --- a/c++/pod5_format/signal_table_schema.h +++ b/c++/pod5_format/signal_table_schema.h @@ -27,11 +27,11 @@ struct SignalTableSchemaDescription { /// \param field_locations [optional] The signal table field locations, for use when writing to the table. /// \returns The schema for a signal table. POD5_FORMAT_EXPORT std::shared_ptr make_signal_table_schema( - SignalType signal_type, - std::shared_ptr const& metadata, - SignalTableSchemaDescription* field_locations); + SignalType signal_type, + std::shared_ptr const & metadata, + SignalTableSchemaDescription * field_locations); POD5_FORMAT_EXPORT Result read_signal_table_schema( - std::shared_ptr const&); + std::shared_ptr const &); } // namespace pod5 diff --git a/c++/pod5_format/signal_table_writer.cpp b/c++/pod5_format/signal_table_writer.cpp index a9aab3b..a4c0765 100644 --- a/c++/pod5_format/signal_table_writer.cpp +++ b/c++/pod5_format/signal_table_writer.cpp @@ -16,19 +16,23 @@ namespace pod5 { -namespace { -namespace visitors { +namespace { namespace visitors { class reserve_rows : boost::static_visitor { public: reserve_rows(std::size_t row_count, std::size_t approx_read_samples) - : m_row_count(row_count), m_approx_read_samples(approx_read_samples) {} + : m_row_count(row_count) + , m_approx_read_samples(approx_read_samples) + { + } - Status operator()(UncompressedSignalBuilder& builder) const { + Status operator()(UncompressedSignalBuilder & builder) const + { ARROW_RETURN_NOT_OK(builder.signal_builder->Reserve(m_row_count)); return builder.signal_data_builder->Reserve(m_row_count * m_approx_read_samples); } - Status operator()(VbzSignalBuilder& builder) const { + Status operator()(VbzSignalBuilder & builder) const + { ARROW_RETURN_NOT_OK(builder.offset_values.reserve(m_row_count + 1)); return builder.data_values.reserve(m_row_count * m_approx_read_samples); } @@ -39,17 +43,19 @@ class reserve_rows : boost::static_visitor { class append_pre_compressed_signal : boost::static_visitor { public: - append_pre_compressed_signal(gsl::span const& signal) : m_signal(signal) {} + append_pre_compressed_signal(gsl::span const & signal) : m_signal(signal) {} - Status operator()(UncompressedSignalBuilder& builder) const { + Status operator()(UncompressedSignalBuilder & builder) const + { ARROW_RETURN_NOT_OK(builder.signal_builder->Append()); // start new slot auto as_uncompressed = m_signal.as_span(); - return builder.signal_data_builder->AppendValues(as_uncompressed.data(), - as_uncompressed.size()); + return builder.signal_data_builder->AppendValues( + as_uncompressed.data(), as_uncompressed.size()); } - Status operator()(VbzSignalBuilder& builder) const { + Status operator()(VbzSignalBuilder & builder) const + { ARROW_RETURN_NOT_OK(builder.offset_values.append(builder.data_values.size())); return builder.data_values.append_array(m_signal); } @@ -59,35 +65,42 @@ class append_pre_compressed_signal : boost::static_visitor { class append_signal : boost::static_visitor { public: - append_signal(gsl::span const& signal, arrow::MemoryPool* pool) - : m_signal(signal), m_pool(pool) {} + append_signal(gsl::span const & signal, arrow::MemoryPool * pool) + : m_signal(signal) + , m_pool(pool) + { + } - Status operator()(UncompressedSignalBuilder& builder) const { + Status operator()(UncompressedSignalBuilder & builder) const + { ARROW_RETURN_NOT_OK(builder.signal_builder->Append()); // start new slot return builder.signal_data_builder->AppendValues(m_signal.data(), m_signal.size()); } - Status operator()(VbzSignalBuilder& builder) const { + Status operator()(VbzSignalBuilder & builder) const + { ARROW_ASSIGN_OR_RAISE(auto compressed_signal, compress_signal(m_signal, m_pool)); ARROW_RETURN_NOT_OK(builder.offset_values.append(builder.data_values.size())); return builder.data_values.append_array( - gsl::make_span(compressed_signal->data(), compressed_signal->size())); + gsl::make_span(compressed_signal->data(), compressed_signal->size())); } gsl::span m_signal; - arrow::MemoryPool* m_pool; + arrow::MemoryPool * m_pool; }; class finish_column : boost::static_visitor { public: - finish_column(std::shared_ptr* dest) : m_dest(dest) {} + finish_column(std::shared_ptr * dest) : m_dest(dest) {} - Status operator()(UncompressedSignalBuilder& builder) const { + Status operator()(UncompressedSignalBuilder & builder) const + { return builder.signal_builder->Finish(m_dest); } - Status operator()(VbzSignalBuilder& builder) const { + Status operator()(VbzSignalBuilder & builder) const + { auto offsets_copy = builder.offset_values; ARROW_RETURN_NOT_OK(builder.offset_values.clear()); @@ -102,50 +115,55 @@ class finish_column : boost::static_visitor { std::shared_ptr null_bitmap; - *m_dest = arrow::MakeArray(arrow::ArrayData::Make( - vbz_signal(), length, {null_bitmap, offsets, value_data}, 0, 0)); + *m_dest = arrow::MakeArray( + arrow::ArrayData::Make(vbz_signal(), length, {null_bitmap, offsets, value_data}, 0, 0)); return arrow::Status::OK(); } - std::shared_ptr* m_dest; + std::shared_ptr * m_dest; }; -} // namespace visitors -} // namespace - -SignalTableWriter::SignalTableWriter(std::shared_ptr&& writer, - std::shared_ptr&& schema, - SignalBuilderVariant&& signal_builder, - SignalTableSchemaDescription const& field_locations, - std::size_t table_batch_size, - arrow::MemoryPool* pool) - : m_pool(pool), - m_schema(schema), - m_field_locations(field_locations), - m_table_batch_size(table_batch_size), - m_writer(std::move(writer)), - m_signal_builder(std::move(signal_builder)) { +}} // namespace ::visitors + +SignalTableWriter::SignalTableWriter( + std::shared_ptr && writer, + std::shared_ptr && schema, + SignalBuilderVariant && signal_builder, + SignalTableSchemaDescription const & field_locations, + std::size_t table_batch_size, + arrow::MemoryPool * pool) +: m_pool(pool) +, m_schema(schema) +, m_field_locations(field_locations) +, m_table_batch_size(table_batch_size) +, m_writer(std::move(writer)) +, m_signal_builder(std::move(signal_builder)) +{ auto uuid_type = m_schema->field(m_field_locations.read_id)->type(); assert(uuid_type->id() == arrow::Type::EXTENSION); auto uuid_extension = std::static_pointer_cast(uuid_type); m_read_id_builder = - std::make_unique(uuid_extension->storage_type(), m_pool); + std::make_unique(uuid_extension->storage_type(), m_pool); assert(m_read_id_builder->byte_width() == 16); m_samples_builder = std::make_unique(m_pool); } -SignalTableWriter::SignalTableWriter(SignalTableWriter&& other) = default; -SignalTableWriter& SignalTableWriter::operator=(SignalTableWriter&&) = default; -SignalTableWriter::~SignalTableWriter() { +SignalTableWriter::SignalTableWriter(SignalTableWriter && other) = default; +SignalTableWriter & SignalTableWriter::operator=(SignalTableWriter &&) = default; + +SignalTableWriter::~SignalTableWriter() +{ if (m_writer) { (void)close(); } } -Result SignalTableWriter::add_signal(boost::uuids::uuid const& read_id, - gsl::span const& signal) { +Result SignalTableWriter::add_signal( + boost::uuids::uuid const & read_id, + gsl::span const & signal) +{ POD5_TRACE_FUNCTION(); if (!m_writer) { return Status::IOError("Writer terminated"); @@ -155,7 +173,7 @@ Result SignalTableWriter::add_signal(boost::uuids::uuid const& read ARROW_RETURN_NOT_OK(m_read_id_builder->Append(read_id.begin())); ARROW_RETURN_NOT_OK( - boost::apply_visitor(visitors::append_signal{signal, m_pool}, m_signal_builder)); + boost::apply_visitor(visitors::append_signal{signal, m_pool}, m_signal_builder)); ARROW_RETURN_NOT_OK(m_samples_builder->Append(signal.size())); ++m_current_batch_row_count; @@ -168,9 +186,10 @@ Result SignalTableWriter::add_signal(boost::uuids::uuid const& read } Result SignalTableWriter::add_pre_compressed_signal( - boost::uuids::uuid const& read_id, - gsl::span const& signal, - std::uint32_t sample_count) { + boost::uuids::uuid const & read_id, + gsl::span const & signal, + std::uint32_t sample_count) +{ POD5_TRACE_FUNCTION(); if (!m_writer) { return Status::IOError("Writer terminated"); @@ -180,7 +199,7 @@ Result SignalTableWriter::add_pre_compressed_signal( ARROW_RETURN_NOT_OK(m_read_id_builder->Append(read_id.begin())); ARROW_RETURN_NOT_OK( - boost::apply_visitor(visitors::append_pre_compressed_signal{signal}, m_signal_builder)); + boost::apply_visitor(visitors::append_pre_compressed_signal{signal}, m_signal_builder)); ARROW_RETURN_NOT_OK(m_samples_builder->Append(sample_count)); ++m_current_batch_row_count; @@ -192,7 +211,8 @@ Result SignalTableWriter::add_pre_compressed_signal( return row_id; } -Status SignalTableWriter::close() { +Status SignalTableWriter::close() +{ // Check for already closed if (!m_writer) { return Status::OK(); @@ -207,7 +227,8 @@ Status SignalTableWriter::close() { SignalType SignalTableWriter::signal_type() const { return m_field_locations.signal_type; } -Status SignalTableWriter::write_batch() { +Status SignalTableWriter::write_batch() +{ POD5_TRACE_FUNCTION(); if (m_current_batch_row_count == 0) { return Status::OK(); @@ -221,12 +242,12 @@ Status SignalTableWriter::write_batch() { ARROW_RETURN_NOT_OK(m_read_id_builder->Finish(&columns[m_field_locations.read_id])); ARROW_RETURN_NOT_OK(boost::apply_visitor( - visitors::finish_column{&columns[m_field_locations.signal]}, m_signal_builder)); + visitors::finish_column{&columns[m_field_locations.signal]}, m_signal_builder)); ARROW_RETURN_NOT_OK(m_samples_builder->Finish(&columns[m_field_locations.samples])); auto const record_batch = - arrow::RecordBatch::Make(m_schema, m_current_batch_row_count, std::move(columns)); + arrow::RecordBatch::Make(m_schema, m_current_batch_row_count, std::move(columns)); m_written_batched_row_count += m_current_batch_row_count; m_current_batch_row_count = 0; @@ -236,22 +257,24 @@ Status SignalTableWriter::write_batch() { return reserve_rows(); } -Status SignalTableWriter::reserve_rows() { +Status SignalTableWriter::reserve_rows() +{ ARROW_RETURN_NOT_OK(m_read_id_builder->Reserve(m_table_batch_size)); ARROW_RETURN_NOT_OK(m_samples_builder->Reserve(m_table_batch_size)); static constexpr std::uint32_t APPROX_READ_SIZE = 102'400; - return boost::apply_visitor(visitors::reserve_rows{m_table_batch_size, APPROX_READ_SIZE}, - m_signal_builder); + return boost::apply_visitor( + visitors::reserve_rows{m_table_batch_size, APPROX_READ_SIZE}, m_signal_builder); } Result make_signal_table_writer( - std::shared_ptr const& sink, - std::shared_ptr const& metadata, - std::size_t table_batch_size, - SignalType compression_type, - arrow::MemoryPool* pool) { + std::shared_ptr const & sink, + std::shared_ptr const & metadata, + std::size_t table_batch_size, + SignalType compression_type, + arrow::MemoryPool * pool) +{ SignalTableSchemaDescription field_locations; auto schema = make_signal_table_schema(compression_type, metadata, &field_locations); @@ -264,8 +287,8 @@ Result make_signal_table_writer( if (compression_type == SignalType::UncompressedSignal) { auto signal_array_builder = std::make_shared(pool); signal_builder = UncompressedSignalBuilder{ - signal_array_builder, - std::make_unique(pool, signal_array_builder), + signal_array_builder, + std::make_unique(pool, signal_array_builder), }; } else { VbzSignalBuilder vbz_builder; @@ -274,9 +297,13 @@ Result make_signal_table_writer( signal_builder = vbz_builder; } - auto signal_table_writer = - SignalTableWriter(std::move(writer), std::move(schema), std::move(signal_builder), - field_locations, table_batch_size, pool); + auto signal_table_writer = SignalTableWriter( + std::move(writer), + std::move(schema), + std::move(signal_builder), + field_locations, + table_batch_size, + pool); ARROW_RETURN_NOT_OK(signal_table_writer.reserve_rows()); return signal_table_writer; diff --git a/c++/pod5_format/signal_table_writer.h b/c++/pod5_format/signal_table_writer.h index 598b6b5..f216531 100644 --- a/c++/pod5_format/signal_table_writer.h +++ b/c++/pod5_format/signal_table_writer.h @@ -12,9 +12,11 @@ namespace arrow { class Schema; + namespace io { class OutputStream; } + namespace ipc { class RecordBatchWriter; } @@ -36,24 +38,26 @@ class POD5_FORMAT_EXPORT SignalTableWriter { public: using SignalBuilderVariant = boost::variant; - SignalTableWriter(std::shared_ptr&& writer, - std::shared_ptr&& schema, - SignalBuilderVariant&& signal_builder, - SignalTableSchemaDescription const& field_locations, - std::size_t table_batch_size, - arrow::MemoryPool* pool); - SignalTableWriter(SignalTableWriter&&); - SignalTableWriter& operator=(SignalTableWriter&&); - SignalTableWriter(SignalTableWriter const&) = delete; - SignalTableWriter& operator=(SignalTableWriter const&) = delete; + SignalTableWriter( + std::shared_ptr && writer, + std::shared_ptr && schema, + SignalBuilderVariant && signal_builder, + SignalTableSchemaDescription const & field_locations, + std::size_t table_batch_size, + arrow::MemoryPool * pool); + SignalTableWriter(SignalTableWriter &&); + SignalTableWriter & operator=(SignalTableWriter &&); + SignalTableWriter(SignalTableWriter const &) = delete; + SignalTableWriter & operator=(SignalTableWriter const &) = delete; ~SignalTableWriter(); /// \brief Add a read to the signal table, adding to the current batch. /// \param read_id The read id for the read entry /// \param signal The signal for the read entry /// \returns The row index of the inserted signal, or a status on failure. - Result add_signal(boost::uuids::uuid const& read_id, - gsl::span const& signal); + Result add_signal( + boost::uuids::uuid const & read_id, + gsl::span const & signal); /// \brief Add a pre-compressed read to the signal table, adding to the current batch. /// The batch is not flushed to disk until #flush is called. @@ -64,9 +68,10 @@ class POD5_FORMAT_EXPORT SignalTableWriter { /// \param read_id The read id for the read entry /// \param signal The signal for the read entry /// \returns The row index of the inserted signal, or a status on failure. - Result add_pre_compressed_signal(boost::uuids::uuid const& read_id, - gsl::span const& signal, - std::uint32_t sample_count); + Result add_pre_compressed_signal( + boost::uuids::uuid const & read_id, + gsl::span const & signal, + std::uint32_t sample_count); /// \brief Close this writer, signaling no further data will be written to the writer. Status close(); @@ -81,7 +86,7 @@ class POD5_FORMAT_EXPORT SignalTableWriter { /// \brief Flush buffered data into the writer as a record batch. Status write_batch(); - arrow::MemoryPool* m_pool = nullptr; + arrow::MemoryPool * m_pool = nullptr; std::shared_ptr m_schema; SignalTableSchemaDescription m_field_locations; std::size_t m_table_batch_size; @@ -103,10 +108,10 @@ class POD5_FORMAT_EXPORT SignalTableWriter { /// \param pool Pool to be used for building table in memory. /// \returns The writer for the new table. POD5_FORMAT_EXPORT Result make_signal_table_writer( - std::shared_ptr const& sink, - std::shared_ptr const& metadata, - std::size_t table_batch_size, - SignalType compression_type, - arrow::MemoryPool* pool); + std::shared_ptr const & sink, + std::shared_ptr const & metadata, + std::size_t table_batch_size, + SignalType compression_type, + arrow::MemoryPool * pool); } // namespace pod5 diff --git a/c++/pod5_format/svb16/common.hpp b/c++/pod5_format/svb16/common.hpp index 430a9c8..43ee830 100644 --- a/c++/pod5_format/svb16/common.hpp +++ b/c++/pod5_format/svb16/common.hpp @@ -27,7 +27,8 @@ #define svb16_popcount __builtin_popcount #else // optimising compilers can often convert this pattern to POPCNT on x86_64 -inline int svb16_popcount(unsigned int i) { +inline int svb16_popcount(unsigned int i) +{ i = i - ((i >> 1) & 0x55555555); // add pairs of bits i = (i & 0x33333333) + ((i >> 2) & 0x33333333); // quads i = (i + (i >> 4)) & 0x0F0F0F0F; // groups of 8 diff --git a/c++/pod5_format/svb16/decode.hpp b/c++/pod5_format/svb16/decode.hpp index 5233cd7..0869aca 100644 --- a/c++/pod5_format/svb16/decode.hpp +++ b/c++/pod5_format/svb16/decode.hpp @@ -11,7 +11,8 @@ namespace svb16 { template -size_t decode(Int16T* out, uint8_t const* SVB_RESTRICT in, uint32_t count, Int16T prev = 0) { +size_t decode(Int16T * out, uint8_t const * SVB_RESTRICT in, uint32_t count, Int16T prev = 0) +{ auto const keys = in; auto const data = keys + ::svb16_key_length(count); #ifdef SVB16_X64 diff --git a/c++/pod5_format/svb16/decode_scalar.hpp b/c++/pod5_format/svb16/decode_scalar.hpp index 50a96b6..15b3798 100644 --- a/c++/pod5_format/svb16/decode_scalar.hpp +++ b/c++/pod5_format/svb16/decode_scalar.hpp @@ -8,11 +8,14 @@ namespace svb16 { namespace detail { -inline uint16_t zigzag_decode(uint16_t val) { +inline uint16_t zigzag_decode(uint16_t val) +{ return (val >> 1) ^ static_cast(0 - (val & 1)); } -inline uint16_t decode_data(uint8_t const *SVB_RESTRICT *dataPtrPtr, uint8_t code) { - const uint8_t *dataPtr = *dataPtrPtr; + +inline uint16_t decode_data(uint8_t const * SVB_RESTRICT * dataPtrPtr, uint8_t code) +{ + uint8_t const * dataPtr = *dataPtrPtr; uint16_t val; if (code == 0) { // 1 byte @@ -30,11 +33,13 @@ inline uint16_t decode_data(uint8_t const *SVB_RESTRICT *dataPtrPtr, uint8_t cod } // namespace detail template -uint8_t const *decode_scalar(Int16T *out, - uint8_t const *SVB_RESTRICT keys, - uint8_t const *SVB_RESTRICT data, - uint32_t count, - Int16T prev = 0) { +uint8_t const * decode_scalar( + Int16T * out, + uint8_t const * SVB_RESTRICT keys, + uint8_t const * SVB_RESTRICT data, + uint32_t count, + Int16T prev = 0) +{ if (count == 0) { return data; } @@ -50,7 +55,8 @@ uint8_t const *decode_scalar(Int16T *out, } uint16_t value = detail::decode_data(&data, (key_byte >> shift) & 0x01); SVB16_IF_CONSTEXPR(UseZigzag) { value = detail::zigzag_decode(value); } - SVB16_IF_CONSTEXPR(UseDelta) { + SVB16_IF_CONSTEXPR(UseDelta) + { value += u_prev; u_prev = value; } diff --git a/c++/pod5_format/svb16/decode_x64.hpp b/c++/pod5_format/svb16/decode_x64.hpp index 9fd1726..b885e0d 100644 --- a/c++/pod5_format/svb16/decode_x64.hpp +++ b/c++/pod5_format/svb16/decode_x64.hpp @@ -13,17 +13,19 @@ namespace svb16 { namespace detail { -[[gnu::target("ssse3")]] inline __m128i zigzag_decode(__m128i val) { +[[gnu::target("ssse3")]] inline __m128i zigzag_decode(__m128i val) +{ return _mm_xor_si128( - // N >> 1 - _mm_srli_epi16(val, 1), - // 0xFFFF if N & 1 else 0x0000 - _mm_srai_epi16(_mm_slli_epi16(val, 15), 15) - // alternative: _mm_sign_epi16(ones, _mm_slli_epi16(buf, 15)) + // N >> 1 + _mm_srli_epi16(val, 1), + // 0xFFFF if N & 1 else 0x0000 + _mm_srai_epi16(_mm_slli_epi16(val, 15), 15) + // alternative: _mm_sign_epi16(ones, _mm_slli_epi16(buf, 15)) ); } -[[gnu::target("ssse3")]] inline __m128i unpack(uint32_t key, const uint8_t *SVB_RESTRICT *data) { +[[gnu::target("ssse3")]] inline __m128i unpack(uint32_t key, uint8_t const * SVB_RESTRICT * data) +{ auto const len = static_cast(8 + svb16_popcount(key)); __m128i data_reg = _mm_loadu_si128(reinterpret_cast<__m128i const *>(*data)); __m128i const shuffle = *reinterpret_cast<__m128i const *>(&g_decode_shuffle_table[key]); @@ -35,12 +37,14 @@ namespace detail { } template -[[gnu::target("ssse3")]] inline void store_8(Int16T *to, __m128i value, __m128i *prev) { +[[gnu::target("ssse3")]] inline void store_8(Int16T * to, __m128i value, __m128i * prev) +{ SVB16_IF_CONSTEXPR(UseZigzag) { value = zigzag_decode(value); } - SVB16_IF_CONSTEXPR(UseDelta) { + SVB16_IF_CONSTEXPR(UseDelta) + { auto const broadcast_last_16 = - m128i_from_bytes(14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15); + m128i_from_bytes(14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15); // value == [A B C D E F G H] (16 bit values) __m128i add = _mm_slli_si128(value, 2); // add == [- A B C D E F G] @@ -66,12 +70,14 @@ template } // namespace detail template -[[gnu::target("sse4.1")]] uint8_t const *decode_sse(Int16T *out, - uint8_t const *SVB_RESTRICT keys, - uint8_t const *SVB_RESTRICT data, - uint32_t count, - Int16T prev = 0) { - auto store_8 = [](Int16T *to, __m128i value, __m128i *prev) { +[[gnu::target("sse4.1")]] uint8_t const * decode_sse( + Int16T * out, + uint8_t const * SVB_RESTRICT keys, + uint8_t const * SVB_RESTRICT data, + uint32_t count, + Int16T prev = 0) +{ + auto store_8 = [](Int16T * to, __m128i value, __m128i * prev) { detail::store_8(to, value, prev); }; // this code treats all input as uint16_t (except the zigzag code, which treats it as int16_t) @@ -85,7 +91,7 @@ template SVB16_IF_CONSTEXPR(UseDelta) { prev_reg = _mm_set1_epi16(prev); } int64_t offset = -static_cast(key_bytes) / 8 + 1; // 8 -> 4? - const uint64_t *keyPtr64 = reinterpret_cast(keys) - offset; + uint64_t const * keyPtr64 = reinterpret_cast(keys) - offset; uint64_t nextkeys; memcpy(&nextkeys, keyPtr64 + offset, sizeof(nextkeys)); @@ -99,28 +105,28 @@ template // _mm_cvtepu8_epi16: SSE4.1 data_reg = - _mm_cvtepu8_epi16(_mm_lddqu_si128(reinterpret_cast<__m128i const *>(data))); + _mm_cvtepu8_epi16(_mm_lddqu_si128(reinterpret_cast<__m128i const *>(data))); store_8(out, data_reg, &prev_reg); - data_reg = _mm_cvtepu8_epi16( - _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 8))); + data_reg = + _mm_cvtepu8_epi16(_mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 8))); store_8(out + 8, data_reg, &prev_reg); data_reg = _mm_cvtepu8_epi16( - _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 16))); + _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 16))); store_8(out + 16, data_reg, &prev_reg); data_reg = _mm_cvtepu8_epi16( - _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 24))); + _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 24))); store_8(out + 24, data_reg, &prev_reg); data_reg = _mm_cvtepu8_epi16( - _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 32))); + _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 32))); store_8(out + 32, data_reg, &prev_reg); data_reg = _mm_cvtepu8_epi16( - _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + +40))); + _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + +40))); store_8(out + 40, data_reg, &prev_reg); data_reg = _mm_cvtepu8_epi16( - _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 48))); + _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 48))); store_8(out + 48, data_reg, &prev_reg); data_reg = _mm_cvtepu8_epi16( - _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 56))); + _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 56))); store_8(out + 56, data_reg, &prev_reg); out += 64; data += 64; @@ -157,28 +163,28 @@ template // faster 16-bit delta since we only have 8-bit values if (!keys) { // 64 1-byte ints in a row data_reg = - _mm_cvtepu8_epi16(_mm_lddqu_si128(reinterpret_cast<__m128i const *>(data))); + _mm_cvtepu8_epi16(_mm_lddqu_si128(reinterpret_cast<__m128i const *>(data))); store_8(out, data_reg, &prev_reg); - data_reg = _mm_cvtepu8_epi16( - _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 8))); + data_reg = + _mm_cvtepu8_epi16(_mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 8))); store_8(out + 8, data_reg, &prev_reg); data_reg = _mm_cvtepu8_epi16( - _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 16))); + _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 16))); store_8(out + 16, data_reg, &prev_reg); data_reg = _mm_cvtepu8_epi16( - _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 24))); + _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 24))); store_8(out + 24, data_reg, &prev_reg); data_reg = _mm_cvtepu8_epi16( - _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 32))); + _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 32))); store_8(out + 32, data_reg, &prev_reg); data_reg = _mm_cvtepu8_epi16( - _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + +40))); + _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + +40))); store_8(out + 40, data_reg, &prev_reg); data_reg = _mm_cvtepu8_epi16( - _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 48))); + _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 48))); store_8(out + 48, data_reg, &prev_reg); data_reg = _mm_cvtepu8_epi16( - _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 56))); + _mm_lddqu_si128(reinterpret_cast<__m128i const *>(data + 56))); store_8(out + 56, data_reg, &prev_reg); out += 64; data += 64; diff --git a/c++/pod5_format/svb16/encode.hpp b/c++/pod5_format/svb16/encode.hpp index 0ef6a3f..131b683 100644 --- a/c++/pod5_format/svb16/encode.hpp +++ b/c++/pod5_format/svb16/encode.hpp @@ -11,7 +11,8 @@ namespace svb16 { template -size_t encode(Int16T const* in, uint8_t* SVB_RESTRICT out, uint32_t count, Int16T prev = 0) { +size_t encode(Int16T const * in, uint8_t * SVB_RESTRICT out, uint32_t count, Int16T prev = 0) +{ auto const keys = out; auto const data = keys + ::svb16_key_length(count); #ifdef SVB16_X64 diff --git a/c++/pod5_format/svb16/encode_scalar.hpp b/c++/pod5_format/svb16/encode_scalar.hpp index 4392988..aa20abf 100644 --- a/c++/pod5_format/svb16/encode_scalar.hpp +++ b/c++/pod5_format/svb16/encode_scalar.hpp @@ -8,17 +8,20 @@ namespace svb16 { namespace detail { -inline uint16_t zigzag_encode(uint16_t val) { +inline uint16_t zigzag_encode(uint16_t val) +{ return (val + val) ^ static_cast(static_cast(val) >> 15); } } // namespace detail template -uint8_t* encode_scalar(Int16T const* in, - uint8_t* SVB_RESTRICT keys, - uint8_t* SVB_RESTRICT data, - uint32_t count, - Int16T prev = 0) { +uint8_t * encode_scalar( + Int16T const * in, + uint8_t * SVB_RESTRICT keys, + uint8_t * SVB_RESTRICT data, + uint32_t count, + Int16T prev = 0) +{ if (count == 0) { return data; } @@ -32,7 +35,8 @@ uint8_t* encode_scalar(Int16T const* in, key_byte = 0; } uint16_t value; - SVB16_IF_CONSTEXPR(UseDelta) { + SVB16_IF_CONSTEXPR(UseDelta) + { // need to do the arithmetic in unsigned space so it wraps value = static_cast(in[c]) - static_cast(prev); SVB16_IF_CONSTEXPR(UseZigzag) { value = detail::zigzag_encode(value); } diff --git a/c++/pod5_format/svb16/encode_x64.hpp b/c++/pod5_format/svb16/encode_x64.hpp index 5108978..26575d9 100644 --- a/c++/pod5_format/svb16/encode_x64.hpp +++ b/c++/pod5_format/svb16/encode_x64.hpp @@ -13,18 +13,22 @@ namespace svb16 { namespace detail { -[[gnu::target("ssse3")]] inline __m128i delta(__m128i curr, __m128i prev) { +[[gnu::target("ssse3")]] inline __m128i delta(__m128i curr, __m128i prev) +{ return _mm_sub_epi16(curr, _mm_alignr_epi8(curr, prev, 14)); } -[[gnu::target("ssse3")]] inline __m128i zigzag_encode(__m128i val) { +[[gnu::target("ssse3")]] inline __m128i zigzag_encode(__m128i val) +{ return _mm_xor_si128(_mm_add_epi16(val, val), _mm_srai_epi16(val, 16)); } template -[[gnu::target("ssse3")]] inline __m128i load_8(Int16T const *from, __m128i *prev) { +[[gnu::target("ssse3")]] inline __m128i load_8(Int16T const * from, __m128i * prev) +{ auto const loaded = _mm_loadu_si128(reinterpret_cast<__m128i const *>(from)); - SVB16_IF_CONSTEXPR(UseDelta && UseZigzag) { + SVB16_IF_CONSTEXPR(UseDelta && UseZigzag) + { auto const result = delta(loaded, *prev); *prev = loaded; return zigzag_encode(result); @@ -44,19 +48,36 @@ template } // namespace detail template -[[gnu::target("ssse3")]] uint8_t *encode_sse(Int16T const *in, - uint8_t *SVB_RESTRICT keys_dest, - uint8_t *SVB_RESTRICT data_dest, - uint32_t count, - Int16T prev = 0) { +[[gnu::target("ssse3")]] uint8_t * encode_sse( + Int16T const * in, + uint8_t * SVB_RESTRICT keys_dest, + uint8_t * SVB_RESTRICT data_dest, + uint32_t count, + Int16T prev = 0) +{ // this code treats all input as uint16_t (except the zigzag code, which treats it as int16_t) // this isn't a problem, as the scalar code does the same __m128i prev_reg; SVB16_IF_CONSTEXPR(UseDelta) { prev_reg = _mm_set1_epi16(prev); } //auto const key_len = svb16_key_length(count); - auto const mask_01 = detail::m128i_from_bytes(0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01); - for (const Int16T *end = &in [(count & ~15)]; in != end; in += 16) { + auto const mask_01 = detail::m128i_from_bytes( + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01, + 0x01); + for (Int16T const * end = &in [(count & ~15)]; in != end; in += 16) { // load up 16 values into r0 and r1 auto r0 = detail::load_8(in, &prev_reg); auto r1 = detail::load_8(in + 8, &prev_reg); diff --git a/c++/pod5_format/svb16/intrinsics.hpp b/c++/pod5_format/svb16/intrinsics.hpp index 6868ab1..632c3aa 100644 --- a/c++/pod5_format/svb16/intrinsics.hpp +++ b/c++/pod5_format/svb16/intrinsics.hpp @@ -12,38 +12,53 @@ #include -namespace svb16 { -namespace detail { -[[gnu::target("sse2")]] inline constexpr __m128i m128i_from_bytes(uint8_t a, - uint8_t b, - uint8_t c, - uint8_t d, - uint8_t e, - uint8_t f, - uint8_t g, - uint8_t h, - uint8_t i, - uint8_t j, - uint8_t k, - uint8_t l, - uint8_t m, - uint8_t n, - uint8_t o, - uint8_t p) { +namespace svb16 { namespace detail { +[[gnu::target("sse2")]] inline constexpr __m128i m128i_from_bytes( + uint8_t a, + uint8_t b, + uint8_t c, + uint8_t d, + uint8_t e, + uint8_t f, + uint8_t g, + uint8_t h, + uint8_t i, + uint8_t j, + uint8_t k, + uint8_t l, + uint8_t m, + uint8_t n, + uint8_t o, + uint8_t p) +{ #ifdef _MSC_VER - return __m128i{(char)a, (char)b, (char)c, (char)d, (char)e, (char)f, (char)g, (char)h, - (char)i, (char)j, (char)k, (char)l, (char)m, (char)n, (char)o, (char)p}; + return __m128i{ + (char)a, + (char)b, + (char)c, + (char)d, + (char)e, + (char)f, + (char)g, + (char)h, + (char)i, + (char)j, + (char)k, + (char)l, + (char)m, + (char)n, + (char)o, + (char)p}; #else return __m128i{ - static_cast(static_cast(h) << 56) + (static_cast(g) << 48) + - (static_cast(f) << 40) + (static_cast(e) << 32) + - (static_cast(d) << 24) + (static_cast(c) << 16) + - (static_cast(b) << 8) + static_cast(a), - static_cast(static_cast(h) << 56) + (static_cast(g) << 48) + - (static_cast(f) << 40) + (static_cast(e) << 32) + - (static_cast(d) << 24) + (static_cast(c) << 16) + - (static_cast(b) << 8) + static_cast(a)}; + static_cast(static_cast(h) << 56) + (static_cast(g) << 48) + + (static_cast(f) << 40) + (static_cast(e) << 32) + + (static_cast(d) << 24) + (static_cast(c) << 16) + + (static_cast(b) << 8) + static_cast(a), + static_cast(static_cast(h) << 56) + (static_cast(g) << 48) + + (static_cast(f) << 40) + (static_cast(e) << 32) + + (static_cast(d) << 24) + (static_cast(c) << 16) + + (static_cast(b) << 8) + static_cast(a)}; #endif } -} // namespace detail -} // namespace svb16 +}} // namespace svb16::detail diff --git a/c++/pod5_format/svb16/shuffle_tables.hpp b/c++/pod5_format/svb16/shuffle_tables.hpp index 70a83ca..730a750 100644 --- a/c++/pod5_format/svb16/shuffle_tables.hpp +++ b/c++/pod5_format/svb16/shuffle_tables.hpp @@ -5,658 +5,4233 @@ #ifdef SVB16_X64 static constexpr uint8_t g_encode_shuffle_table[128 * 16] = { - 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0x00, 0x01, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0C, 0x0E, - 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0C, - 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, - 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x04, 0x06, 0x07, 0x08, 0x0A, - 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, 0x06, 0x07, - 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x06, - 0x07, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, - 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x04, - 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, - 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, - 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, - 0xFF, 0x00, 0x02, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0E, - 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0E, - 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, - 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x09, - 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, - 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x04, 0x06, 0x07, 0x08, - 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, 0x06, - 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, - 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, - 0x03, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0x00, 0x02, - 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, - 0x01, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, - 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, - 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, - 0xFF, 0xFF, 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, - 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x08, 0x0A, 0x0B, 0x0C, - 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0B, 0x0C, - 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x0A, - 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, - 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, - 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x04, 0x06, 0x07, - 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, - 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x03, - 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, - 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0x00, - 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, - 0xFF, 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, - 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, - 0x0F, 0xFF, 0xFF, 0x00, 0x02, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, - 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x08, 0x09, 0x0A, - 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, - 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, - 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, - 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0x00, 0x02, 0x04, 0x06, - 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, - 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0x00, 0x02, - 0x03, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0x00, - 0x01, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, - 0x00, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, - 0xFF, 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, - 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, - 0x0F, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0x0C, 0x0E, 0x0F, 0xFF, 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, - 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, - 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0D, - 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x08, 0x0A, - 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x04, 0x05, 0x06, 0x08, 0x0A, - 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, - 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x05, - 0x06, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, - 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x04, - 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, - 0x02, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, - 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, - 0xFF, 0x00, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, - 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, - 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, - 0x0F, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0C, - 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0x00, 0x02, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, - 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0C, - 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x06, 0x08, 0x09, 0x0A, - 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x08, - 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x04, 0x05, 0x06, 0x08, - 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, 0x05, - 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, - 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, - 0x03, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0x00, 0x02, - 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, - 0x01, 0x02, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, - 0x00, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, - 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, - 0xFF, 0xFF, 0x00, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, - 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0D, - 0x0E, 0x0F, 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, - 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, - 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, - 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0B, - 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x06, 0x08, 0x0A, - 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, - 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x04, 0x05, 0x06, - 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, - 0x05, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x03, - 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0x00, 0x01, - 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0x00, - 0x02, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, - 0x00, 0x01, 0x02, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, - 0xFF, 0x00, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, - 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, - 0x0F, 0xFF, 0xFF, 0x00, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, - 0x0F, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, - 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0B, - 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, - 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0x00, 0x02, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0B, - 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, 0x06, 0x08, 0x09, - 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x06, 0x08, - 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, - 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0x00, 0x02, 0x04, 0x05, - 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0x00, 0x01, 0x02, - 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0x00, 0x02, - 0x03, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0x00, - 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, - 0x00, 0x02, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, - 0xFF, 0x00, 0x01, 0x02, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, - 0xFF, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, - 0x0F, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0x0D, 0x0E, 0x0F, 0xFF, 0x00, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, - 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, - 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, - 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0E, 0x0F, 0xFF, + 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, + 0x00, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, + 0x00, 0x02, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, + 0x00, 0x02, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, + 0x00, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, 0xFF, + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, + 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0xFF, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, }; static const uint8_t g_decode_shuffle_table[256][16] = { - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0xFF, 0x0A, 0xFF, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0xFF, 0x0A, 0xFF, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, 0xFF, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, 0xFF, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, 0xFF, 0x0B, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0xFF, 0x0B, 0xFF, 0x0C, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0x0A, 0x0B, 0xFF, 0x0C, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xFF, 0x0C, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xFF, 0x0C, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xFF, 0x0C, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xFF, 0x0C, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0xFF, 0x0D, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0xFF, 0x0A, 0x0B, 0x0C, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0xFF, 0x0A, 0x0B, 0x0C, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, 0x0B, 0x0C, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, 0x0B, 0x0C, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, 0x0B, 0x0C, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0xFF, 0x0B, 0x0C, 0x0D, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0xFF}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, - 0xFF}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, - 0xFF}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, - 0xFF}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, - 0x08}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, - 0x09}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, - 0x09}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0x0A}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, - 0x09}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0x0A}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0x0A}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, - 0x09}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0x0A}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0x0A}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0x0A}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0xFF, 0x0A, 0xFF, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, - 0x09}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0x0A}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0x0A}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0x0A}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0xFF, 0x0A, 0xFF, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, - 0x0A}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, 0xFF, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, 0xFF, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, 0xFF, 0x0B, - 0x0C}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0xFF, 0x0B, 0xFF, 0x0C, - 0x0D}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, - 0x09}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, - 0x0A}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, - 0x0A}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, - 0x0A}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, - 0x0A}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0x0C}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0x0A, 0x0B, 0xFF, 0x0C, - 0x0D}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, - 0x0A}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0x0C}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xFF, 0x0C, - 0x0D}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0x0C}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xFF, 0x0C, - 0x0D}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0xFF, 0x0B, - 0x0C}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xFF, 0x0C, - 0x0D}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0xFF, 0x0C, - 0x0D}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0xFF, 0x0D, - 0x0E}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, - 0x09}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, - 0x0A}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, - 0x0A}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0x0B}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, - 0x0A}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0x0B}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, - 0x0A}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0x0B}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0xFF, 0x0A, 0x0B, 0x0C, - 0x0D}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, - 0x0A}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0x0B}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0xFF, 0x0A, 0x0B, 0x0C, - 0x0D}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, 0x0B, 0x0C, - 0x0D}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, 0x0B, 0x0C, - 0x0D}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0xFF, 0x0A, 0x0B, 0x0C, - 0x0D}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0xFF, 0x0B, 0x0C, 0x0D, - 0x0E}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, - 0x0A}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, - 0x0B}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0xFF, 0x06, 0xFF, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0xFF, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0x0D}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0x0D}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0x0D}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0xFF, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0x0D}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0xFF, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, - 0x0E}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, - 0x0B}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0x0D}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0x0D}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0xFF, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0x0D}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0xFF, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, - 0x0E}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, - 0x0C}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0x0D}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0xFF, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0x0D}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0xFF, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, - 0x0E}, - {0x00, 0xFF, 0x01, 0xFF, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, - 0x0D}, - {0x00, 0x01, 0x02, 0xFF, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, - 0x0E}, - {0x00, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, - 0x0E}, - {0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, - 0x0F}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0xFF, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0xFF, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0xFF, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0xFF, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0xFF, + 0x0B, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0xFF, + 0x0C, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0xFF, + 0x0C, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0xFF, + 0x0C, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0xFF, + 0x0C, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0xFF, + 0x0C, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0xFF, + 0x0C, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0xFF, + 0x0D, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0x0B, + 0x0C, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0x0B, + 0x0C, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0x0B, + 0x0C, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0x0B, + 0x0C, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0x0B, + 0x0C, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0x0C, + 0x0D, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D, + 0xFF}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D, + 0x0E, + 0xFF}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0x0A}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0x0A}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0x0A}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0x0A}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0x0A}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0x0A}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0xFF, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0x0A}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0x0A}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0x0A}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0xFF, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0x0A}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0xFF, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0xFF, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0xFF, + 0x0B, + 0x0C}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0xFF, + 0x0C, + 0x0D}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0x0C}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0xFF, + 0x0C, + 0x0D}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0x0C}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0xFF, + 0x0C, + 0x0D}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0x0C}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0xFF, + 0x0C, + 0x0D}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0x0C}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0xFF, + 0x0C, + 0x0D}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0xFF, + 0x0C, + 0x0D}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0xFF, + 0x0D, + 0x0E}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0x0B}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0x0B}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0x0B}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0xFF, + 0x0A, + 0x0B, + 0x0C, + 0x0D}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0x0B}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0x0B, + 0x0C, + 0x0D}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0x0B, + 0x0C, + 0x0D}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0x0B, + 0x0C, + 0x0D}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0xFF, + 0x0A, + 0x0B, + 0x0C, + 0x0D}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0xFF, + 0x0B, + 0x0C, + 0x0D, + 0x0E}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0xFF, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0xFF, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D, + 0x0E}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0xFF, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0xFF, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D, + 0x0E}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0xFF, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0xFF, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D, + 0x0E}, + {0x00, + 0xFF, + 0x01, + 0xFF, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D}, + {0x00, + 0x01, + 0x02, + 0xFF, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D, + 0x0E}, + {0x00, + 0xFF, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D, + 0x0E}, + {0x00, + 0x01, + 0x02, + 0x03, + 0x04, + 0x05, + 0x06, + 0x07, + 0x08, + 0x09, + 0x0A, + 0x0B, + 0x0C, + 0x0D, + 0x0E, + 0x0F}, }; #endif diff --git a/c++/pod5_format/svb16/simd_detect_x64.hpp b/c++/pod5_format/svb16/simd_detect_x64.hpp index c99e050..497df8a 100644 --- a/c++/pod5_format/svb16/simd_detect_x64.hpp +++ b/c++/pod5_format/svb16/simd_detect_x64.hpp @@ -24,15 +24,16 @@ struct CpuidResult { unsigned int edx; }; -inline CpuidResult cpuid(unsigned int leaf, unsigned int subleaf) { +inline CpuidResult cpuid(unsigned int leaf, unsigned int subleaf) +{ #ifdef _MSC_VER int info[4]; __cpuidex(info, static_cast(leaf), static_cast(subleaf)); return CpuidResult{ - static_cast(info[0]), - static_cast(info[1]), - static_cast(info[2]), - static_cast(info[3]), + static_cast(info[0]), + static_cast(info[1]), + static_cast(info[2]), + static_cast(info[3]), }; #else CpuidResult info; @@ -43,9 +44,10 @@ inline CpuidResult cpuid(unsigned int leaf, unsigned int subleaf) { #endif } -inline unsigned int cpuid_leaf1_ecx() { +inline unsigned int cpuid_leaf1_ecx() +{ // using C++11 atomic static variables - static const unsigned int ecx = cpuid(1, 0).ecx; + static unsigned int const ecx = cpuid(1, 0).ecx; return ecx; } diff --git a/c++/pod5_format/svb16/streamvbytedelta_decode_16.c b/c++/pod5_format/svb16/streamvbytedelta_decode_16.c index 138a23d..7d220c1 100644 --- a/c++/pod5_format/svb16/streamvbytedelta_decode_16.c +++ b/c++/pod5_format/svb16/streamvbytedelta_decode_16.c @@ -3,12 +3,14 @@ #include // for memcpy -static inline uint16_t zigzag_decode_16(uint16_t val) { +static inline uint16_t zigzag_decode_16(uint16_t val) +{ return (val >> 1) ^ (uint16_t)(0 - (val & 1)); } -static inline uint16_t _decode_data(const uint8_t **dataPtrPtr, uint8_t code) { - const uint8_t *dataPtr = *dataPtrPtr; +static inline uint16_t _decode_data(uint8_t const ** dataPtrPtr, uint8_t code) +{ + uint8_t const * dataPtr = *dataPtrPtr; uint16_t val; if (code == 0) { // 1 byte @@ -24,11 +26,13 @@ static inline uint16_t _decode_data(const uint8_t **dataPtrPtr, uint8_t code) { return val; } -static const uint8_t *svb_decode_scalar_d1_init(uint16_t *outPtr, - const uint8_t *keyPtr, - const uint8_t *dataPtr, - uint32_t count, - uint16_t prev) { +static uint8_t const * svb_decode_scalar_d1_init( + uint16_t * outPtr, + uint8_t const * keyPtr, + uint8_t const * dataPtr, + uint32_t count, + uint16_t prev) +{ if (count == 0) return dataPtr; // no reads or writes if no data @@ -55,14 +59,16 @@ static const uint8_t *svb_decode_scalar_d1_init(uint16_t *outPtr, #include "streamvbytedelta_x64_decode_16.c" #endif -size_t streamvbyte_zigzag_delta_decode_16(const uint8_t *in, - uint16_t *out, - uint32_t count, - uint16_t prev) { +size_t streamvbyte_zigzag_delta_decode_16( + uint8_t const * in, + uint16_t * out, + uint32_t count, + uint16_t prev) +{ // keyLen = ceil(count / 8), without overflowing (1 bit per input value): uint32_t keyLen = (count >> 3) + (((count & 7) + 7) >> 3); - const uint8_t *keyPtr = in; - const uint8_t *dataPtr = keyPtr + keyLen; // data starts at end of keys + uint8_t const * keyPtr = in; + uint8_t const * dataPtr = keyPtr + keyLen; // data starts at end of keys #ifdef STREAMVBYTE_X64 if (streamvbyte_ssse3()) { return svb_decode_avx_d1_init(out, keyPtr, dataPtr, count, prev) - in; diff --git a/c++/pod5_format/svb16/streamvbytedelta_encode_16.c b/c++/pod5_format/svb16/streamvbytedelta_encode_16.c index fa97f8a..0c4cd3f 100644 --- a/c++/pod5_format/svb16/streamvbytedelta_encode_16.c +++ b/c++/pod5_format/svb16/streamvbytedelta_encode_16.c @@ -8,12 +8,14 @@ #include "streamvbytedelta_x64_encode_16.c" #endif -static inline uint16_t _zigzag_encode_16(uint16_t val) { +static inline uint16_t _zigzag_encode_16(uint16_t val) +{ return (val + val) ^ ((int16_t)val >> 15); } -static uint8_t _encode_data(uint16_t val, uint8_t *__restrict__ *dataPtrPtr) { - uint8_t *dataPtr = *dataPtrPtr; +static uint8_t _encode_data(uint16_t val, uint8_t * __restrict__ * dataPtrPtr) +{ + uint8_t * dataPtr = *dataPtrPtr; uint8_t code; if (val < (1 << 8)) { // 1 byte @@ -29,11 +31,13 @@ static uint8_t _encode_data(uint16_t val, uint8_t *__restrict__ *dataPtrPtr) { return code; } -static uint8_t *svb_encode_scalar_d1_init(const uint16_t *in, - uint8_t *__restrict__ keyPtr, - uint8_t *__restrict__ dataPtr, - uint32_t count, - uint16_t prev) { +static uint8_t * svb_encode_scalar_d1_init( + uint16_t const * in, + uint8_t * __restrict__ keyPtr, + uint8_t * __restrict__ dataPtr, + uint32_t count, + uint16_t prev) +{ if (count == 0) return dataPtr; // exit immediately if no data @@ -57,18 +61,20 @@ static uint8_t *svb_encode_scalar_d1_init(const uint16_t *in, return dataPtr; // pointer to first unused data byte } -size_t streamvbyte_zigzag_delta_encode_16(const uint16_t *in, - uint32_t count, - uint8_t *out, - uint16_t prev) { +size_t streamvbyte_zigzag_delta_encode_16( + uint16_t const * in, + uint32_t count, + uint8_t * out, + uint16_t prev) +{ #ifdef STREAMVBYTE_X64 if (streamvbyte_ssse3()) { return streamvbyte_zigzag_delta_encode_SSSE3_d1_init(in, count, out, prev); } #endif - uint8_t *keyPtr = out; // keys come at start + uint8_t * keyPtr = out; // keys come at start // keyLen = ceil(count / 8), without overflowing (1 bit per input value): uint32_t keyLen = (count >> 3) + (((count & 7) + 7) >> 3); - uint8_t *dataPtr = keyPtr + keyLen; // variable byte data after all keys + uint8_t * dataPtr = keyPtr + keyLen; // variable byte data after all keys return svb_encode_scalar_d1_init(in, keyPtr, dataPtr, count, prev) - out; } diff --git a/c++/pod5_format/svb16/streamvbytedelta_x64_decode_16.c b/c++/pod5_format/svb16/streamvbytedelta_x64_decode_16.c index 95ffbbc..0f0f661 100644 --- a/c++/pod5_format/svb16/streamvbytedelta_x64_decode_16.c +++ b/c++/pod5_format/svb16/streamvbytedelta_x64_decode_16.c @@ -5,19 +5,22 @@ #ifdef STREAMVBYTE_X64 STREAMVBYTE_TARGET_SSSE3 -static __m128i undo_zigzag_16(__m128i buf) { +static __m128i undo_zigzag_16(__m128i buf) +{ return _mm_xor_si128( - // N >> 1 - _mm_srli_epi16(buf, 1), - // 0xFFFF if N & 1 else 0x0000 - _mm_srai_epi16(_mm_slli_epi16(buf, 15), 15) - // alternative: _mm_sign_epi16(ones, _mm_slli_epi16(buf, 15)) + // N >> 1 + _mm_srli_epi16(buf, 1), + // 0xFFFF if N & 1 else 0x0000 + _mm_srai_epi16(_mm_slli_epi16(buf, 15), 15) + // alternative: _mm_sign_epi16(ones, _mm_slli_epi16(buf, 15)) ); } + STREAMVBYTE_UNTARGET_REGION STREAMVBYTE_TARGET_SSSE3 -static inline __m128i _decode_avx(uint32_t key, const uint8_t *__restrict__ *dataPtrPtr) { +static inline __m128i _decode_avx(uint32_t key, uint8_t const * __restrict__ * dataPtrPtr) +{ uint8_t len = 8 + popcount(key); __m128i Data = _mm_loadu_si128((__m128i *)*dataPtrPtr); __m128i Shuf = *(__m128i *)&shuffleTable[key]; @@ -27,14 +30,20 @@ static inline __m128i _decode_avx(uint32_t key, const uint8_t *__restrict__ *dat return Data; } + STREAMVBYTE_UNTARGET_REGION STREAMVBYTE_TARGET_SSSE3 -static inline void _write_avx(uint16_t *out, __m128i Vec) { _mm_storeu_si128((__m128i *)out, Vec); } +static inline void _write_avx(uint16_t * out, __m128i Vec) +{ + _mm_storeu_si128((__m128i *)out, Vec); +} + STREAMVBYTE_UNTARGET_REGION STREAMVBYTE_TARGET_SSSE3 -static inline __m128i _write_16bit_avx_d1(uint16_t *out, __m128i Vec, __m128i Prev) { +static inline __m128i _write_16bit_avx_d1(uint16_t * out, __m128i Vec, __m128i Prev) +{ #ifndef _MSC_VER __m128i BroadcastLast16 = {0x0F0E0F0E0F0E0F0E, 0x0F0E0F0E0F0E0F0E}; #else @@ -53,14 +62,17 @@ static inline __m128i _write_16bit_avx_d1(uint16_t *out, __m128i Vec, __m128i Pr _write_avx(out, Vec); return Vec; } + STREAMVBYTE_UNTARGET_REGION STREAMVBYTE_TARGET_SSSE3 -static const uint8_t *svb_decode_avx_d1_init(uint16_t *out, - const uint8_t *__restrict__ keyPtr, - const uint8_t *__restrict__ dataPtr, - uint64_t count, - uint16_t prev) { +static uint8_t const * svb_decode_avx_d1_init( + uint16_t * out, + uint8_t const * __restrict__ keyPtr, + uint8_t const * __restrict__ dataPtr, + uint64_t count, + uint16_t prev) +{ uint64_t keybytes = count / 4; // number of key bytes if (keybytes >= 8) { __m128i Prev = _mm_set1_epi16(prev); @@ -68,7 +80,7 @@ static const uint8_t *svb_decode_avx_d1_init(uint16_t *out, int64_t Offset = -(int64_t)keybytes / 8 + 1; - const uint64_t *keyPtr64 = (const uint64_t *)keyPtr - Offset; + uint64_t const * keyPtr64 = (uint64_t const *)keyPtr - Offset; uint64_t nextkeys; memcpy(&nextkeys, keyPtr64 + Offset, sizeof(nextkeys)); for (; Offset != 0; ++Offset) { @@ -163,5 +175,6 @@ static const uint8_t *svb_decode_avx_d1_init(uint16_t *out, uint64_t consumedkeys = keybytes - (keybytes & 7); return svb_decode_scalar_d1_init(out, keyPtr + consumedkeys, dataPtr, count & 31, prev); } + STREAMVBYTE_UNTARGET_REGION #endif diff --git a/c++/pod5_format/svb16/streamvbytedelta_x64_encode_16.c b/c++/pod5_format/svb16/streamvbytedelta_x64_encode_16.c index b5199f8..203c1e0 100644 --- a/c++/pod5_format/svb16/streamvbytedelta_x64_encode_16.c +++ b/c++/pod5_format/svb16/streamvbytedelta_x64_encode_16.c @@ -8,32 +8,38 @@ #ifdef STREAMVBYTE_X64 STREAMVBYTE_TARGET_SSSE3 -static __m128i Delta(__m128i curr, __m128i prev) { +static __m128i Delta(__m128i curr, __m128i prev) +{ // _mm_alignr_epi8: SSSE3 return _mm_sub_epi16(curr, _mm_alignr_epi8(curr, prev, 14)); } + STREAMVBYTE_UNTARGET_REGION STREAMVBYTE_TARGET_SSSE3 -static __m128i zigzag_16(__m128i buf) { +static __m128i zigzag_16(__m128i buf) +{ return _mm_xor_si128(_mm_add_epi16(buf, buf), _mm_srai_epi16(buf, 16)); } + STREAMVBYTE_UNTARGET_REGION // based on code by aqrit (streamvbyte_encode_SSSE3) STREAMVBYTE_TARGET_SSSE3 -size_t streamvbyte_zigzag_delta_encode_SSSE3_d1_init(const uint16_t *in, - uint32_t count, - uint8_t *out, - uint16_t prev) { +size_t streamvbyte_zigzag_delta_encode_SSSE3_d1_init( + uint16_t const * in, + uint32_t count, + uint8_t * out, + uint16_t prev) +{ __m128i Prev = _mm_set1_epi16(prev); uint32_t keyLen = (count >> 3) + (((count & 7) + 7) >> 3); // 1-bit rounded to full byte - uint8_t *restrict keyPtr = &out[0]; - uint8_t *restrict dataPtr = &out[keyLen]; // variable length data after keys + uint8_t * restrict keyPtr = &out[0]; + uint8_t * restrict dataPtr = &out[keyLen]; // variable length data after keys const __m128i mask_01 = _mm_set1_epi8(0x01); - for (const uint16_t *end = &in [(count & ~15)]; in != end; in += 16) { + for (uint16_t const * end = &in [(count & ~15)]; in != end; in += 16) { __m128i rawr0, r0, rawr1, r1, r2, r3; size_t keys; @@ -84,5 +90,6 @@ size_t streamvbyte_zigzag_delta_encode_SSSE3_d1_init(const uint16_t *in, return dataPtr - out; } + STREAMVBYTE_UNTARGET_REGION #endif diff --git a/c++/pod5_format/svb16/svb16.h b/c++/pod5_format/svb16/svb16.h index 1ed4104..164cb5f 100644 --- a/c++/pod5_format/svb16/svb16.h +++ b/c++/pod5_format/svb16/svb16.h @@ -8,13 +8,15 @@ extern "C" { #endif /// Get the number of key bytes required to encode a given number of 16-bit integers. -inline uint32_t svb16_key_length(uint32_t count) { +inline uint32_t svb16_key_length(uint32_t count) +{ // ceil(count / 8.0), without overflowing or using fp arithmetic return (count >> 3) + (((count & 7) + 7) >> 3); } /// Get the maximum number of bytes required to encode a given number of 16-bit integers. -inline uint32_t svb16_max_encoded_length(uint32_t count) { +inline uint32_t svb16_max_encoded_length(uint32_t count) +{ return svb16_key_length(count) + (2 * count); } diff --git a/c++/pod5_format/table_reader.cpp b/c++/pod5_format/table_reader.cpp index 56f7bef..d281dc2 100644 --- a/c++/pod5_format/table_reader.cpp +++ b/c++/pod5_format/table_reader.cpp @@ -5,29 +5,34 @@ namespace pod5 { -TableRecordBatch::TableRecordBatch(std::shared_ptr&& batch) - : m_batch(std::move(batch)) {} - -TableRecordBatch::TableRecordBatch(TableRecordBatch const&) = default; -TableRecordBatch& TableRecordBatch::operator=(TableRecordBatch const&) = default; -TableRecordBatch::TableRecordBatch(TableRecordBatch&&) = default; -TableRecordBatch& TableRecordBatch::operator=(TableRecordBatch&&) = default; +TableRecordBatch::TableRecordBatch(std::shared_ptr && batch) +: m_batch(std::move(batch)) +{ +} + +TableRecordBatch::TableRecordBatch(TableRecordBatch const &) = default; +TableRecordBatch & TableRecordBatch::operator=(TableRecordBatch const &) = default; +TableRecordBatch::TableRecordBatch(TableRecordBatch &&) = default; +TableRecordBatch & TableRecordBatch::operator=(TableRecordBatch &&) = default; TableRecordBatch::~TableRecordBatch() = default; std::size_t TableRecordBatch::num_rows() const { return m_batch->num_rows(); } //--------------------------------------------------------------------------------------------------------------------- -TableReader::TableReader(std::shared_ptr&& input_source, - std::shared_ptr&& reader, - SchemaMetadataDescription&& schema_metadata, - arrow::MemoryPool* pool) - : m_input_source(std::move(input_source)), - m_reader(std::move(reader)), - m_schema_metadata(std::move(schema_metadata)) {} - -TableReader::TableReader(TableReader&&) = default; -TableReader& TableReader::operator=(TableReader&&) = default; +TableReader::TableReader( + std::shared_ptr && input_source, + std::shared_ptr && reader, + SchemaMetadataDescription && schema_metadata, + arrow::MemoryPool * pool) +: m_input_source(std::move(input_source)) +, m_reader(std::move(reader)) +, m_schema_metadata(std::move(schema_metadata)) +{ +} + +TableReader::TableReader(TableReader &&) = default; +TableReader & TableReader::operator=(TableReader &&) = default; TableReader::~TableReader() = default; std::size_t TableReader::num_record_batches() const { return m_reader->num_record_batches(); } diff --git a/c++/pod5_format/table_reader.h b/c++/pod5_format/table_reader.h index fa77e15..b74d73d 100644 --- a/c++/pod5_format/table_reader.h +++ b/c++/pod5_format/table_reader.h @@ -8,6 +8,7 @@ namespace arrow { class MemoryPool; class RecordBatch; + namespace ipc { class RecordBatchFileReader; } @@ -17,17 +18,17 @@ namespace pod5 { class POD5_FORMAT_EXPORT TableRecordBatch { public: - TableRecordBatch(std::shared_ptr&& batch); + TableRecordBatch(std::shared_ptr && batch); - TableRecordBatch(TableRecordBatch&&); - TableRecordBatch& operator=(TableRecordBatch&&); - TableRecordBatch(TableRecordBatch const&); - TableRecordBatch& operator=(TableRecordBatch const&); + TableRecordBatch(TableRecordBatch &&); + TableRecordBatch & operator=(TableRecordBatch &&); + TableRecordBatch(TableRecordBatch const &); + TableRecordBatch & operator=(TableRecordBatch const &); ~TableRecordBatch(); std::size_t num_rows() const; - std::shared_ptr const& batch() const { return m_batch; } + std::shared_ptr const & batch() const { return m_batch; } private: std::shared_ptr m_batch; @@ -35,21 +36,22 @@ class POD5_FORMAT_EXPORT TableRecordBatch { class POD5_FORMAT_EXPORT TableReader { public: - TableReader(std::shared_ptr&& input_source, - std::shared_ptr&& reader, - SchemaMetadataDescription&& schema_metadata, - arrow::MemoryPool* pool); - TableReader(TableReader&&); - TableReader& operator=(TableReader&&); - TableReader(TableReader const&) = delete; - TableReader& operator=(TableReader const&) = delete; + TableReader( + std::shared_ptr && input_source, + std::shared_ptr && reader, + SchemaMetadataDescription && schema_metadata, + arrow::MemoryPool * pool); + TableReader(TableReader &&); + TableReader & operator=(TableReader &&); + TableReader(TableReader const &) = delete; + TableReader & operator=(TableReader const &) = delete; ~TableReader(); - SchemaMetadataDescription const& schema_metadata() const { return m_schema_metadata; } + SchemaMetadataDescription const & schema_metadata() const { return m_schema_metadata; } std::size_t num_record_batches() const; - std::shared_ptr const& reader() const { return m_reader; } + std::shared_ptr const & reader() const { return m_reader; } private: std::shared_ptr m_input_source; diff --git a/c++/pod5_format/tuple_utils.h b/c++/pod5_format/tuple_utils.h index 0948eff..034c154 100644 --- a/c++/pod5_format/tuple_utils.h +++ b/c++/pod5_format/tuple_utils.h @@ -3,34 +3,35 @@ #include #include -namespace pod5 { - -namespace detail { +namespace pod5 { namespace detail { template -void for_each(T&& t, F f, std::integer_sequence) { +void for_each(T && t, F f, std::integer_sequence) +{ auto l = {(f(std::get(t), Is), 0)...}; (void)l; } template -void for_each_in_tuple(std::tuple& t, F f) { +void for_each_in_tuple(std::tuple & t, F f) +{ detail::for_each(t, f, std::make_integer_sequence()); } template -void for_each_zipped(T1&& t1, T2&& t2, F f, std::integer_sequence) { +void for_each_zipped(T1 && t1, T2 && t2, F f, std::integer_sequence) +{ auto l = {(f(std::get(t1), std::get(t2), Is), 0)...}; (void)l; } template -void for_each_in_tuple_zipped(T1& t1, T2& t2, F f) { - static_assert(std::tuple_size::value == std::tuple_size::value, - "Tuples must be same size"); - detail::for_each_zipped(t1, t2, f, - std::make_integer_sequence::value>()); +void for_each_in_tuple_zipped(T1 & t1, T2 & t2, F f) +{ + static_assert( + std::tuple_size::value == std::tuple_size::value, "Tuples must be same size"); + detail::for_each_zipped( + t1, t2, f, std::make_integer_sequence::value>()); } -} // namespace detail -} // namespace pod5 +}} // namespace pod5::detail diff --git a/c++/pod5_format/types.cpp b/c++/pod5_format/types.cpp index ac6996c..1d13d79 100644 --- a/c++/pod5_format/types.cpp +++ b/c++/pod5_format/types.cpp @@ -5,41 +5,50 @@ namespace pod5 { -const boost::uuids::uuid *UuidArray::raw_values() const { +boost::uuids::uuid const * UuidArray::raw_values() const +{ auto const array = std::static_pointer_cast(storage()); return reinterpret_cast(array->GetValue(0)); } -boost::uuids::uuid UuidArray::Value(int64_t i) const { +boost::uuids::uuid UuidArray::Value(int64_t i) const +{ auto const array = std::static_pointer_cast(storage()); return *reinterpret_cast(array->GetValue(i)); } -bool UuidType::ExtensionEquals(const ExtensionType &other) const { +bool UuidType::ExtensionEquals(ExtensionType const & other) const +{ // no parameters to consider return other.extension_name() == extension_name(); } -std::shared_ptr UuidType::MakeArray(std::shared_ptr data) const { + +std::shared_ptr UuidType::MakeArray(std::shared_ptr data) const +{ DCHECK_EQ(data->type->id(), arrow::Type::EXTENSION); - DCHECK_EQ(static_cast(*data->type).extension_name(), - extension_name()); + DCHECK_EQ( + static_cast(*data->type).extension_name(), extension_name()); return std::make_shared(data); } + std::string UuidType::Serialize() const { return ""; } + arrow::Result> UuidType::Deserialize( - std::shared_ptr storage_type, - const std::string &serialized_data) const { + std::shared_ptr storage_type, + std::string const & serialized_data) const +{ if (serialized_data != "") { return arrow::Status::Invalid("Unexpected type metadata: '", serialized_data, "'"); } if (!storage_type->Equals(*arrow::fixed_size_binary(16))) { - return arrow::Status::Invalid("Incorrect storage for UuidType: '", storage_type->ToString(), - "'"); + return arrow::Status::Invalid( + "Incorrect storage for UuidType: '", storage_type->ToString(), "'"); } return std::make_shared(); } -gsl::span VbzSignalArray::Value(int64_t i) const { +gsl::span VbzSignalArray::Value(int64_t i) const +{ auto const array = std::static_pointer_cast(storage()); arrow::LargeBinaryArray::offset_type value_length = 0; @@ -47,7 +56,8 @@ gsl::span VbzSignalArray::Value(int64_t i) const { return gsl::make_span(value_ptr, value_length); } -std::shared_ptr VbzSignalArray::ValueAsBuffer(int64_t i) const { +std::shared_ptr VbzSignalArray::ValueAsBuffer(int64_t i) const +{ auto const array = std::static_pointer_cast(storage()); auto offset = array->value_offset(i); @@ -57,43 +67,52 @@ std::shared_ptr VbzSignalArray::ValueAsBuffer(int64_t i) const { return arrow::SliceBuffer(value_data, offset, length); } -bool VbzSignalType::ExtensionEquals(const ExtensionType &other) const { +bool VbzSignalType::ExtensionEquals(ExtensionType const & other) const +{ // no parameters to consider return other.extension_name() == extension_name(); } -std::shared_ptr VbzSignalType::MakeArray( - std::shared_ptr data) const { + +std::shared_ptr VbzSignalType::MakeArray(std::shared_ptr data) const +{ DCHECK_EQ(data->type->id(), arrow::Type::EXTENSION); - DCHECK_EQ(static_cast(*data->type).extension_name(), - extension_name()); + DCHECK_EQ( + static_cast(*data->type).extension_name(), extension_name()); return std::make_shared(data); } + std::string VbzSignalType::Serialize() const { return ""; } + arrow::Result> VbzSignalType::Deserialize( - std::shared_ptr storage_type, - const std::string &serialized_data) const { + std::shared_ptr storage_type, + std::string const & serialized_data) const +{ if (serialized_data != "") { return arrow::Status::Invalid("Unexpected type metadata: '", serialized_data, "'"); } if (!storage_type->Equals(*arrow::large_binary())) { - return arrow::Status::Invalid("Incorrect storage for VbzSignalType: '", - storage_type->ToString(), "'"); + return arrow::Status::Invalid( + "Incorrect storage for VbzSignalType: '", storage_type->ToString(), "'"); } return std::make_shared(); } -std::shared_ptr vbz_signal() { +std::shared_ptr vbz_signal() +{ static auto vbz_signal = std::make_shared(); return vbz_signal; } -std::shared_ptr uuid() { + +std::shared_ptr uuid() +{ static auto uuid = std::make_shared(); return uuid; } std::atomic g_pod5_register_count(0); -pod5::Status register_extension_types() { +pod5::Status register_extension_types() +{ if (++g_pod5_register_count == 1) { ARROW_RETURN_NOT_OK(arrow::RegisterExtensionType(uuid())); ARROW_RETURN_NOT_OK(arrow::RegisterExtensionType(vbz_signal())); @@ -101,7 +120,8 @@ pod5::Status register_extension_types() { return pod5::Status::OK(); } -pod5::Status unregister_extension_types() { +pod5::Status unregister_extension_types() +{ auto register_count = --g_pod5_register_count; if (register_count == 0) { if (arrow::GetExtensionType("minknow.uuid")) { diff --git a/c++/pod5_format/types.h b/c++/pod5_format/types.h index dd84105..a657279 100644 --- a/c++/pod5_format/types.h +++ b/c++/pod5_format/types.h @@ -16,7 +16,7 @@ class POD5_FORMAT_EXPORT UuidArray : public arrow::ExtensionArray { using ExtensionArray::ExtensionArray; - const boost::uuids::uuid *raw_values() const; + boost::uuids::uuid const * raw_values() const; boost::uuids::uuid Value(int64_t i) const; @@ -26,11 +26,13 @@ class POD5_FORMAT_EXPORT UuidArray : public arrow::ExtensionArray { // to be configurable via the ValueAccessor template parameter) boost::uuids::uuid GetView(int64_t i) const { return Value(i); } - arrow::util::optional operator[](int64_t i) const { + arrow::util::optional operator[](int64_t i) const + { return *IteratorType(*this, i); } IteratorType begin() const { return IteratorType(*this); } + IteratorType end() const { return IteratorType(*this, length()); } }; @@ -39,12 +41,13 @@ class POD5_FORMAT_EXPORT UuidType : public arrow::ExtensionType { UuidType() : ExtensionType(arrow::fixed_size_binary(16)) {} std::string extension_name() const override { return "minknow.uuid"; } - bool ExtensionEquals(const ExtensionType &other) const override; + + bool ExtensionEquals(ExtensionType const & other) const override; std::shared_ptr MakeArray(std::shared_ptr data) const override; std::string Serialize() const override; arrow::Result> Deserialize( - std::shared_ptr storage_type, - const std::string &serialized_data) const override; + std::shared_ptr storage_type, + std::string const & serialized_data) const override; }; class POD5_FORMAT_EXPORT VbzSignalArray : public arrow::ExtensionArray { @@ -62,12 +65,13 @@ class POD5_FORMAT_EXPORT VbzSignalType : public arrow::ExtensionType { VbzSignalType() : ExtensionType(arrow::large_binary()) {} std::string extension_name() const override { return "minknow.vbz"; } - bool ExtensionEquals(const ExtensionType &other) const override; + + bool ExtensionEquals(ExtensionType const & other) const override; std::shared_ptr MakeArray(std::shared_ptr data) const override; std::string Serialize() const override; arrow::Result> Deserialize( - std::shared_ptr storage_type, - const std::string &serialized_data) const override; + std::shared_ptr storage_type, + std::string const & serialized_data) const override; }; std::shared_ptr vbz_signal(); diff --git a/c++/pod5_format_pybind/api.h b/c++/pod5_format_pybind/api.h index ad67e24..8343140 100644 --- a/c++/pod5_format_pybind/api.h +++ b/c++/pod5_format_pybind/api.h @@ -19,33 +19,42 @@ namespace py = pybind11; -inline std::shared_ptr create_file(char const* path, - std::string const& writer_name, - pod5::FileWriterOptions const* options) { +inline std::shared_ptr create_file( + char const * path, + std::string const & writer_name, + pod5::FileWriterOptions const * options) +{ pod5::FileWriterOptions dummy; POD5_PYTHON_ASSIGN_OR_RAISE( - auto writer, pod5::create_file_writer(path, writer_name, - options ? *options : pod5::FileWriterOptions{})); + auto writer, + pod5::create_file_writer( + path, writer_name, options ? *options : pod5::FileWriterOptions{})); return writer; } class Pod5SignalCacheBatch { public: - Pod5SignalCacheBatch(pod5::AsyncSignalLoader::SamplesMode samples_mode, - pod5::CachedBatchSignalData&& cached_data) - : m_samples_mode(samples_mode), m_cached_data(std::move(cached_data)) {} + Pod5SignalCacheBatch( + pod5::AsyncSignalLoader::SamplesMode samples_mode, + pod5::CachedBatchSignalData && cached_data) + : m_samples_mode(samples_mode) + , m_cached_data(std::move(cached_data)) + { + } - py::array_t sample_count() const { - return py::array_t(m_cached_data.sample_count().size(), - m_cached_data.sample_count().data()); + py::array_t sample_count() const + { + return py::array_t( + m_cached_data.sample_count().size(), m_cached_data.sample_count().data()); } - py::list samples() const { + py::list samples() const + { py::list py_samples; if (m_samples_mode != pod5::AsyncSignalLoader::SamplesMode::Samples) { return py_samples; } - for (auto const& row_samples : m_cached_data.samples()) { + for (auto const & row_samples : m_cached_data.samples()) { py_samples.append(py::array_t(row_samples.size(), row_samples.data())); } @@ -62,50 +71,60 @@ class Pod5SignalCacheBatch { class Pod5AsyncSignalLoader { public: // Make an async loader for all reads in the file - Pod5AsyncSignalLoader(std::shared_ptr const& reader, - pod5::AsyncSignalLoader::SamplesMode samples_mode, - std::size_t worker_count = std::thread::hardware_concurrency(), - std::size_t max_pending_batches = 10) - : m_samples_mode(samples_mode), - m_batch_counts_ref({}), - m_batch_rows_ref({}), - m_async_loader(reader, samples_mode, {}, {}, worker_count, max_pending_batches) {} + Pod5AsyncSignalLoader( + std::shared_ptr const & reader, + pod5::AsyncSignalLoader::SamplesMode samples_mode, + std::size_t worker_count = std::thread::hardware_concurrency(), + std::size_t max_pending_batches = 10) + : m_samples_mode(samples_mode) + , m_batch_counts_ref({}) + , m_batch_rows_ref({}) + , m_async_loader(reader, samples_mode, {}, {}, worker_count, max_pending_batches) + { + } // Make an async loader for specific batches Pod5AsyncSignalLoader( - std::shared_ptr const& reader, - pod5::AsyncSignalLoader::SamplesMode samples_mode, - py::array_t&& batches, - std::size_t worker_count = std::thread::hardware_concurrency(), - std::size_t max_pending_batches = 10) - : m_samples_mode(samples_mode), - m_batch_sizes(make_batch_counts(reader, batches)), - m_async_loader(reader, - samples_mode, - gsl::make_span(m_batch_sizes), - {}, - worker_count, - max_pending_batches) {} + std::shared_ptr const & reader, + pod5::AsyncSignalLoader::SamplesMode samples_mode, + py::array_t && batches, + std::size_t worker_count = std::thread::hardware_concurrency(), + std::size_t max_pending_batches = 10) + : m_samples_mode(samples_mode) + , m_batch_sizes(make_batch_counts(reader, batches)) + , m_async_loader( + reader, + samples_mode, + gsl::make_span(m_batch_sizes), + {}, + worker_count, + max_pending_batches) + { + } // Make an async loader for specific reads in specific batches Pod5AsyncSignalLoader( - std::shared_ptr const& reader, - pod5::AsyncSignalLoader::SamplesMode samples_mode, - py::array_t&& batch_counts, - py::array_t&& batch_rows, - std::size_t worker_count = std::thread::hardware_concurrency(), - std::size_t max_pending_batches = 10) - : m_samples_mode(samples_mode), - m_batch_counts_ref(std::move(batch_counts)), - m_batch_rows_ref(std::move(batch_rows)), - m_async_loader(reader, - samples_mode, - gsl::make_span(m_batch_counts_ref.data(), m_batch_counts_ref.size()), - gsl::make_span(m_batch_rows_ref.data(), m_batch_rows_ref.size()), - worker_count, - max_pending_batches) {} - - std::shared_ptr release_next_batch() { + std::shared_ptr const & reader, + pod5::AsyncSignalLoader::SamplesMode samples_mode, + py::array_t && batch_counts, + py::array_t && batch_rows, + std::size_t worker_count = std::thread::hardware_concurrency(), + std::size_t max_pending_batches = 10) + : m_samples_mode(samples_mode) + , m_batch_counts_ref(std::move(batch_counts)) + , m_batch_rows_ref(std::move(batch_rows)) + , m_async_loader( + reader, + samples_mode, + gsl::make_span(m_batch_counts_ref.data(), m_batch_counts_ref.size()), + gsl::make_span(m_batch_rows_ref.data(), m_batch_rows_ref.size()), + worker_count, + max_pending_batches) + { + } + + std::shared_ptr release_next_batch() + { auto batch = m_async_loader.release_next_batch(); if (!batch.ok()) { throw std::runtime_error(batch.status().ToString()); @@ -120,14 +139,15 @@ class Pod5AsyncSignalLoader { } std::vector make_batch_counts( - std::shared_ptr const& reader, - py::array_t const& batches) { + std::shared_ptr const & reader, + py::array_t const & batches) + { std::vector batch_counts(reader->num_read_record_batches(), 0); - for (auto const& batch_idx : gsl::make_span(batches.data(), batches.shape(0))) { + for (auto const & batch_idx : gsl::make_span(batches.data(), batches.shape(0))) { auto read_batch = reader->read_read_record_batch(batch_idx); if (!read_batch.ok()) { - throw std::runtime_error("Failed to query read batch count: " + - read_batch.status().ToString()); + throw std::runtime_error( + "Failed to query read batch count: " + read_batch.status().ToString()); } batch_counts[batch_idx] = read_batch->num_rows(); @@ -145,206 +165,215 @@ class Pod5AsyncSignalLoader { struct Pod5FileReaderPtr { std::shared_ptr reader = nullptr; - Pod5FileReaderPtr(std::shared_ptr&& reader_) : reader(std::move(reader_)) {} + Pod5FileReaderPtr(std::shared_ptr && reader_) : reader(std::move(reader_)) {} - pod5::FileLocation get_file_run_info_table_location() const { + pod5::FileLocation get_file_run_info_table_location() const + { return reader->run_info_table_location(); } - pod5::FileLocation get_file_read_table_location() const { + pod5::FileLocation get_file_read_table_location() const + { return reader->read_table_location(); } - pod5::FileLocation get_file_signal_table_location() const { + pod5::FileLocation get_file_signal_table_location() const + { return reader->signal_table_location(); } - std::string get_file_version_pre_migration() const { + std::string get_file_version_pre_migration() const + { return reader->file_version_pre_migration().to_string(); } void close() { reader = nullptr; } std::size_t plan_traversal( - py::array_t const& - read_id_data, - py::array_t& batch_counts, - py::array_t& batch_rows) { + py::array_t const & read_id_data, + py::array_t & batch_counts, + py::array_t & batch_rows) + { auto const read_id_count = read_id_data.shape(0); auto search_input = pod5::ReadIdSearchInput(gsl::make_span( - reinterpret_cast(read_id_data.data()), read_id_count)); + reinterpret_cast(read_id_data.data()), read_id_count)); POD5_PYTHON_ASSIGN_OR_RAISE( - auto find_success_count, - reader->search_for_read_ids( - search_input, - gsl::make_span(batch_counts.mutable_data(), - reader->num_read_record_batches()), - gsl::make_span(batch_rows.mutable_data(), read_id_count))); + auto find_success_count, + reader->search_for_read_ids( + search_input, + gsl::make_span(batch_counts.mutable_data(), reader->num_read_record_batches()), + gsl::make_span(batch_rows.mutable_data(), read_id_count))); return find_success_count; } - std::shared_ptr batch_get_signal(bool get_samples, - bool get_sample_count) { + std::shared_ptr batch_get_signal(bool get_samples, bool get_sample_count) + { return std::make_shared( - reader, get_samples ? pod5::AsyncSignalLoader::SamplesMode::Samples - : pod5::AsyncSignalLoader::SamplesMode::NoSamples); + reader, + get_samples ? pod5::AsyncSignalLoader::SamplesMode::Samples + : pod5::AsyncSignalLoader::SamplesMode::NoSamples); } std::shared_ptr batch_get_signal_batches( - bool get_samples, - bool get_sample_count, - py::array_t&& batches) { + bool get_samples, + bool get_sample_count, + py::array_t && batches) + { return std::make_shared( - reader, - get_samples ? pod5::AsyncSignalLoader::SamplesMode::Samples - : pod5::AsyncSignalLoader::SamplesMode::NoSamples, - std::move(batches)); + reader, + get_samples ? pod5::AsyncSignalLoader::SamplesMode::Samples + : pod5::AsyncSignalLoader::SamplesMode::NoSamples, + std::move(batches)); } std::shared_ptr batch_get_signal_selection( - bool get_samples, - bool get_sample_count, - py::array_t&& batch_counts, - py::array_t&& batch_rows) { + bool get_samples, + bool get_sample_count, + py::array_t && batch_counts, + py::array_t && batch_rows) + { return std::make_shared( - reader, - get_samples ? pod5::AsyncSignalLoader::SamplesMode::Samples - : pod5::AsyncSignalLoader::SamplesMode::NoSamples, - std::move(batch_counts), std::move(batch_rows)); + reader, + get_samples ? pod5::AsyncSignalLoader::SamplesMode::Samples + : pod5::AsyncSignalLoader::SamplesMode::NoSamples, + std::move(batch_counts), + std::move(batch_rows)); } }; -inline Pod5FileReaderPtr open_file(char const* filename) { +inline Pod5FileReaderPtr open_file(char const * filename) +{ POD5_PYTHON_ASSIGN_OR_RAISE(auto reader, pod5::open_file_reader(filename, {})); return Pod5FileReaderPtr(std::move(reader)); } -inline void write_updated_file_to_dest(Pod5FileReaderPtr source, char const* dest_filename) { +inline void write_updated_file_to_dest(Pod5FileReaderPtr source, char const * dest_filename) +{ POD5_PYTHON_RETURN_NOT_OK( - pod5::update_file(arrow::default_memory_pool(), source.reader, dest_filename)); + pod5::update_file(arrow::default_memory_pool(), source.reader, dest_filename)); } inline pod5::RunInfoDictionaryIndex FileWriter_add_run_info( - pod5::FileWriter& w, - std::string& acquisition_id, - std::int64_t acquisition_start_time, - std::int16_t adc_max, - std::int16_t adc_min, - std::vector>&& context_tags, - std::string& experiment_name, - std::string& flow_cell_id, - std::string& flow_cell_product_code, - std::string& protocol_name, - std::string& protocol_run_id, - std::int64_t protocol_start_time, - std::string& sample_id, - std::uint16_t sample_rate, - std::string& sequencing_kit, - std::string& sequencer_position, - std::string& sequencer_position_type, - std::string& software, - std::string& system_name, - std::string& system_type, - std::vector>&& tracking_id) { - return throw_on_error(w.add_run_info({std::move(acquisition_id), - acquisition_start_time, - adc_max, - adc_min, - std::move(context_tags), - std::move(experiment_name), - std::move(flow_cell_id), - std::move(flow_cell_product_code), - std::move(protocol_name), - std::move(protocol_run_id), - std::move(protocol_start_time), - std::move(sample_id), - sample_rate, - std::move(sequencing_kit), - std::move(sequencer_position), - std::move(sequencer_position_type), - std::move(software), - std::move(system_name), - std::move(system_type), - std::move(tracking_id)})); + pod5::FileWriter & w, + std::string & acquisition_id, + std::int64_t acquisition_start_time, + std::int16_t adc_max, + std::int16_t adc_min, + std::vector> && context_tags, + std::string & experiment_name, + std::string & flow_cell_id, + std::string & flow_cell_product_code, + std::string & protocol_name, + std::string & protocol_run_id, + std::int64_t protocol_start_time, + std::string & sample_id, + std::uint16_t sample_rate, + std::string & sequencing_kit, + std::string & sequencer_position, + std::string & sequencer_position_type, + std::string & software, + std::string & system_name, + std::string & system_type, + std::vector> && tracking_id) +{ + return throw_on_error(w.add_run_info( + {std::move(acquisition_id), + acquisition_start_time, + adc_max, + adc_min, + std::move(context_tags), + std::move(experiment_name), + std::move(flow_cell_id), + std::move(flow_cell_product_code), + std::move(protocol_name), + std::move(protocol_run_id), + std::move(protocol_start_time), + std::move(sample_id), + sample_rate, + std::move(sequencing_kit), + std::move(sequencer_position), + std::move(sequencer_position_type), + std::move(software), + std::move(system_name), + std::move(system_type), + std::move(tracking_id)})); } inline pod5::ReadData make_read_data( - std::size_t row_id, - py::array_t const& read_id_data, - py::array_t const& read_numbers, - py::array_t const& start_samples, - py::array_t const& channels, - py::array_t const& wells, - py::array_t const& pore_types, - py::array_t const& calibration_offsets, - py::array_t const& calibration_scales, - py::array_t const& median_befores, - py::array_t const& end_reasons, - py::array_t const& end_reason_forceds, - py::array_t const& run_infos, - py::array_t const& - num_minknow_events, - py::array_t const& tracked_scaling_scale, - py::array_t const& tracked_scaling_shift, - py::array_t const& - predicted_scaling_scale, - py::array_t const& - predicted_scaling_shift, - py::array_t const& - num_reads_since_mux_change, - py::array_t const& - time_since_mux_change) { - auto read_ids = reinterpret_cast(read_id_data.data(0)); - return pod5::ReadData{read_ids[row_id], - *read_numbers.data(row_id), - *start_samples.data(row_id), - *channels.data(row_id), - *wells.data(row_id), - *pore_types.data(row_id), - *calibration_offsets.data(row_id), - *calibration_scales.data(row_id), - *median_befores.data(row_id), - *end_reasons.data(row_id), - *end_reason_forceds.data(row_id), - *run_infos.data(row_id), - *num_minknow_events.data(row_id), - *tracked_scaling_scale.data(row_id), - *tracked_scaling_shift.data(row_id), - *predicted_scaling_scale.data(row_id), - *predicted_scaling_shift.data(row_id), - *num_reads_since_mux_change.data(row_id), - *time_since_mux_change.data(row_id)}; + std::size_t row_id, + py::array_t const & read_id_data, + py::array_t const & read_numbers, + py::array_t const & start_samples, + py::array_t const & channels, + py::array_t const & wells, + py::array_t const & pore_types, + py::array_t const & calibration_offsets, + py::array_t const & calibration_scales, + py::array_t const & median_befores, + py::array_t const & end_reasons, + py::array_t const & end_reason_forceds, + py::array_t const & run_infos, + py::array_t const & + num_minknow_events, + py::array_t const & tracked_scaling_scale, + py::array_t const & tracked_scaling_shift, + py::array_t const & predicted_scaling_scale, + py::array_t const & predicted_scaling_shift, + py::array_t const & + num_reads_since_mux_change, + py::array_t const & time_since_mux_change) +{ + auto read_ids = reinterpret_cast(read_id_data.data(0)); + return pod5::ReadData{ + read_ids[row_id], + *read_numbers.data(row_id), + *start_samples.data(row_id), + *channels.data(row_id), + *wells.data(row_id), + *pore_types.data(row_id), + *calibration_offsets.data(row_id), + *calibration_scales.data(row_id), + *median_befores.data(row_id), + *end_reasons.data(row_id), + *end_reason_forceds.data(row_id), + *run_infos.data(row_id), + *num_minknow_events.data(row_id), + *tracked_scaling_scale.data(row_id), + *tracked_scaling_shift.data(row_id), + *predicted_scaling_scale.data(row_id), + *predicted_scaling_shift.data(row_id), + *num_reads_since_mux_change.data(row_id), + *time_since_mux_change.data(row_id)}; } inline void FileWriter_add_reads( - pod5::FileWriter& w, - std::size_t count, - py::array_t const& read_id_data, - py::array_t const& read_numbers, - py::array_t const& start_samples, - py::array_t const& channels, - py::array_t const& wells, - py::array_t const& pore_types, - py::array_t const& calibration_offsets, - py::array_t const& calibration_scales, - py::array_t const& median_befores, - py::array_t const& end_reasons, - py::array_t const& end_reason_forceds, - py::array_t const& run_infos, - py::array_t const& - num_minknow_events, - py::array_t const& tracked_scaling_scales, - py::array_t const& tracked_scaling_shifts, - py::array_t const& - predicted_scaling_scales, - py::array_t const& - predicted_scaling_shifts, - py::array_t const& - num_reads_since_mux_changes, - py::array_t const& time_since_mux_changes, - py::list signal_ptrs) { + pod5::FileWriter & w, + std::size_t count, + py::array_t const & read_id_data, + py::array_t const & read_numbers, + py::array_t const & start_samples, + py::array_t const & channels, + py::array_t const & wells, + py::array_t const & pore_types, + py::array_t const & calibration_offsets, + py::array_t const & calibration_scales, + py::array_t const & median_befores, + py::array_t const & end_reasons, + py::array_t const & end_reason_forceds, + py::array_t const & run_infos, + py::array_t const & + num_minknow_events, + py::array_t const & tracked_scaling_scales, + py::array_t const & tracked_scaling_shifts, + py::array_t const & predicted_scaling_scales, + py::array_t const & predicted_scaling_shifts, + py::array_t const & + num_reads_since_mux_changes, + py::array_t const & time_since_mux_changes, + py::list signal_ptrs) +{ if (read_id_data.shape(1) != 16) { throw std::runtime_error("Read id array is of unexpected size"); } @@ -354,56 +383,70 @@ inline void FileWriter_add_reads( if (signal_it == signal_ptrs.end()) { throw std::runtime_error("Missing signal data"); } - auto signal = signal_it->cast< - py::array_t>(); + auto signal = + signal_it->cast>(); auto signal_span = gsl::make_span(signal.data(), signal.size()); auto read_data = make_read_data( - i, read_id_data, read_numbers, start_samples, channels, wells, pore_types, - calibration_offsets, calibration_scales, median_befores, end_reasons, - end_reason_forceds, run_infos, num_minknow_events, tracked_scaling_scales, - tracked_scaling_shifts, predicted_scaling_scales, predicted_scaling_shifts, - num_reads_since_mux_changes, time_since_mux_changes); + i, + read_id_data, + read_numbers, + start_samples, + channels, + wells, + pore_types, + calibration_offsets, + calibration_scales, + median_befores, + end_reasons, + end_reason_forceds, + run_infos, + num_minknow_events, + tracked_scaling_scales, + tracked_scaling_shifts, + predicted_scaling_scales, + predicted_scaling_shifts, + num_reads_since_mux_changes, + time_since_mux_changes); throw_on_error(w.add_complete_read(read_data, signal_span)); } } inline void FileWriter_add_reads_pre_compressed( - pod5::FileWriter& w, - std::size_t count, - py::array_t const& read_id_data, - py::array_t const& read_numbers, - py::array_t const& start_samples, - py::array_t const& channels, - py::array_t const& wells, - py::array_t const& pore_types, - py::array_t const& calibration_offsets, - py::array_t const& calibration_scales, - py::array_t const& median_befores, - py::array_t const& end_reasons, - py::array_t const& end_reason_forceds, - py::array_t const& run_infos, - py::array_t const& - num_minknow_events, - py::array_t const& tracked_scaling_scales, - py::array_t const& tracked_scaling_shifts, - py::array_t const& - predicted_scaling_scales, - py::array_t const& - predicted_scaling_shifts, - py::array_t const& - num_reads_since_mux_changes, - py::array_t const& time_since_mux_changes, - py::list compressed_signal_ptrs, - py::array_t const& sample_counts, - py::array_t const& - signal_chunk_counts) { + pod5::FileWriter & w, + std::size_t count, + py::array_t const & read_id_data, + py::array_t const & read_numbers, + py::array_t const & start_samples, + py::array_t const & channels, + py::array_t const & wells, + py::array_t const & pore_types, + py::array_t const & calibration_offsets, + py::array_t const & calibration_scales, + py::array_t const & median_befores, + py::array_t const & end_reasons, + py::array_t const & end_reason_forceds, + py::array_t const & run_infos, + py::array_t const & + num_minknow_events, + py::array_t const & tracked_scaling_scales, + py::array_t const & tracked_scaling_shifts, + py::array_t const & predicted_scaling_scales, + py::array_t const & predicted_scaling_shifts, + py::array_t const & + num_reads_since_mux_changes, + py::array_t const & time_since_mux_changes, + py::list compressed_signal_ptrs, + py::array_t const & sample_counts, + py::array_t const & + signal_chunk_counts) +{ if (read_id_data.shape(1) != 16) { throw std::runtime_error("Read id array is of unexpected size"); } - auto read_ids = reinterpret_cast(read_id_data.data(0)); + auto read_ids = reinterpret_cast(read_id_data.data(0)); auto compressed_signal_it = compressed_signal_ptrs.begin(); auto sample_counts_it = sample_counts.data(); for (std::size_t i = 0; i < count; ++i) { @@ -417,13 +460,14 @@ inline void FileWriter_add_reads_pre_compressed( if (compressed_signal_it == compressed_signal_ptrs.end()) { throw std::runtime_error("Missing signal data"); } - auto compressed_signal = compressed_signal_it->cast< - py::array_t>(); + auto compressed_signal = + compressed_signal_it + ->cast>(); auto compressed_signal_span = - gsl::make_span(compressed_signal.data(), compressed_signal.size()); + gsl::make_span(compressed_signal.data(), compressed_signal.size()); - auto signal_row = throw_on_error(w.add_pre_compressed_signal( - read_id, compressed_signal_span, *sample_counts_it)); + auto signal_row = throw_on_error( + w.add_pre_compressed_signal(read_id, compressed_signal_span, *sample_counts_it)); signal_rows[signal_chunk_idx] = signal_row; signal_duration_count += *sample_counts_it; @@ -432,49 +476,68 @@ inline void FileWriter_add_reads_pre_compressed( } auto read_data = make_read_data( - i, read_id_data, read_numbers, start_samples, channels, wells, pore_types, - calibration_offsets, calibration_scales, median_befores, end_reasons, - end_reason_forceds, run_infos, num_minknow_events, tracked_scaling_scales, - tracked_scaling_shifts, predicted_scaling_scales, predicted_scaling_shifts, - num_reads_since_mux_changes, time_since_mux_changes); + i, + read_id_data, + read_numbers, + start_samples, + channels, + wells, + pore_types, + calibration_offsets, + calibration_scales, + median_befores, + end_reasons, + end_reason_forceds, + run_infos, + num_minknow_events, + tracked_scaling_scales, + tracked_scaling_shifts, + predicted_scaling_scales, + predicted_scaling_shifts, + num_reads_since_mux_changes, + time_since_mux_changes); throw_on_error(w.add_complete_read(read_data, signal_rows, signal_duration_count)); } } inline void decompress_signal_wrapper( - py::array_t const& compressed_signal, - py::array_t& signal_out) { + py::array_t const & compressed_signal, + py::array_t & signal_out) +{ throw_on_error(pod5::decompress_signal( - gsl::make_span(compressed_signal.data(0), compressed_signal.shape(0)), - arrow::system_memory_pool(), - gsl::make_span(signal_out.mutable_data(0), signal_out.shape(0)))); + gsl::make_span(compressed_signal.data(0), compressed_signal.shape(0)), + arrow::system_memory_pool(), + gsl::make_span(signal_out.mutable_data(0), signal_out.shape(0)))); } inline std::size_t compress_signal_wrapper( - py::array_t const& signal, - py::array_t& - compressed_signal_out) { + py::array_t const & signal, + py::array_t & compressed_signal_out) +{ auto size = throw_on_error(pod5::compress_signal( - gsl::make_span(signal.data(), signal.shape(0)), arrow::system_memory_pool(), - gsl::make_span(compressed_signal_out.mutable_data(), compressed_signal_out.shape(0)))); + gsl::make_span(signal.data(), signal.shape(0)), + arrow::system_memory_pool(), + gsl::make_span(compressed_signal_out.mutable_data(), compressed_signal_out.shape(0)))); return size; } -inline std::size_t vbz_compressed_signal_max_size(std::size_t sample_count) { +inline std::size_t vbz_compressed_signal_max_size(std::size_t sample_count) +{ return pod5::compressed_signal_max_size(sample_count); } inline std::size_t load_read_id_iterable( - py::iterable const& read_ids_str, - py::array_t& read_id_data_out) { + py::iterable const & read_ids_str, + py::array_t & read_id_data_out) +{ std::size_t out_idx = 0; - auto read_ids = reinterpret_cast(read_id_data_out.mutable_data()); + auto read_ids = reinterpret_cast(read_id_data_out.mutable_data()); auto read_ids_out_len = read_id_data_out.shape(0); std::string temp_uuid; - for (auto& read_id : read_ids_str) { + for (auto & read_id : read_ids_str) { if (out_idx >= read_ids_out_len) { throw std::runtime_error("Too many input uuids for output container"); } @@ -484,7 +547,7 @@ inline std::size_t load_read_id_iterable( auto found_uuid = boost::lexical_cast(temp_uuid); read_ids[out_idx++] = found_uuid; - } catch (boost::bad_lexical_cast const& e) { + } catch (boost::bad_lexical_cast const & e) { // Ignore - we will return one fewer read ids than expected and the caller can deal with it. continue; } @@ -494,10 +557,11 @@ inline std::size_t load_read_id_iterable( } inline py::list format_read_id_to_str( - py::array_t& read_id_data_out) { + py::array_t & read_id_data_out) +{ if (read_id_data_out.size() % 16 != 0) { throw std::runtime_error( - "Unexpected amount of data for read id - expected data to align to 16 bytes."); + "Unexpected amount of data for read id - expected data to align to 16 bytes."); } py::list result; diff --git a/c++/pod5_format_pybind/bindings.cpp b/c++/pod5_format_pybind/bindings.cpp index 14e637b..bb849a9 100644 --- a/c++/pod5_format_pybind/bindings.cpp +++ b/c++/pod5_format_pybind/bindings.cpp @@ -2,77 +2,94 @@ #include "pod5_format/c_api.h" #include "repacker.h" -PYBIND11_MODULE(pod5_format_pybind, m) { +PYBIND11_MODULE(pod5_format_pybind, m) +{ using namespace pod5; pod5_init(); m.doc() = "POD5 Format Raw Bindings"; py::class_(m, "FileWriterOptions") - .def_property("max_signal_chunk_size", &FileWriterOptions::max_signal_chunk_size, - &FileWriterOptions::set_max_signal_chunk_size) - .def_property("signal_table_batch_size", &FileWriterOptions::signal_table_batch_size, - &FileWriterOptions::set_signal_table_batch_size) - .def_property("read_table_batch_size", &FileWriterOptions::read_table_batch_size, - &FileWriterOptions::set_read_table_batch_size) - .def_property("signal_compression_type", &FileWriterOptions::signal_type, - &FileWriterOptions::set_signal_type); + .def_property( + "max_signal_chunk_size", + &FileWriterOptions::max_signal_chunk_size, + &FileWriterOptions::set_max_signal_chunk_size) + .def_property( + "signal_table_batch_size", + &FileWriterOptions::signal_table_batch_size, + &FileWriterOptions::set_signal_table_batch_size) + .def_property( + "read_table_batch_size", + &FileWriterOptions::read_table_batch_size, + &FileWriterOptions::set_read_table_batch_size) + .def_property( + "signal_compression_type", + &FileWriterOptions::signal_type, + &FileWriterOptions::set_signal_type); py::class_>(m, "FileWriter") - .def("close", [](pod5::FileWriter& w) { throw_on_error(w.close()); }) - .def("add_pore", - [](pod5::FileWriter& w, std::string pore_type) { - return throw_on_error(w.add_pore_type(std::move(pore_type))); - }) - .def("add_end_reason", - [](pod5::FileWriter& w, int name) { - return throw_on_error(w.lookup_end_reason((pod5::ReadEndReason)name)); - }) - .def("add_run_info", FileWriter_add_run_info) - .def("add_reads", FileWriter_add_reads) - .def("add_reads_pre_compressed", FileWriter_add_reads_pre_compressed); + .def("close", [](pod5::FileWriter & w) { throw_on_error(w.close()); }) + .def( + "add_pore", + [](pod5::FileWriter & w, std::string pore_type) { + return throw_on_error(w.add_pore_type(std::move(pore_type))); + }) + .def( + "add_end_reason", + [](pod5::FileWriter & w, int name) { + return throw_on_error(w.lookup_end_reason((pod5::ReadEndReason)name)); + }) + .def("add_run_info", FileWriter_add_run_info) + .def("add_reads", FileWriter_add_reads) + .def("add_reads_pre_compressed", FileWriter_add_reads_pre_compressed); py::class_(m, "EmbeddedFileData") - .def_readonly("file_path", &pod5::FileLocation::file_path) - .def_readonly("offset", &pod5::FileLocation::offset) - .def_readonly("length", &pod5::FileLocation::size); + .def_readonly("file_path", &pod5::FileLocation::file_path) + .def_readonly("offset", &pod5::FileLocation::offset) + .def_readonly("length", &pod5::FileLocation::size); py::class_>( - m, "Pod5AsyncSignalLoader") - .def("release_next_batch", &Pod5AsyncSignalLoader::release_next_batch); + m, "Pod5AsyncSignalLoader") + .def("release_next_batch", &Pod5AsyncSignalLoader::release_next_batch); - py::class_>(m, - "Pod5SignalCacheBatch") - .def_property_readonly("batch_index", &Pod5SignalCacheBatch::batch_index) - .def_property_readonly("sample_count", &Pod5SignalCacheBatch::sample_count) - .def_property_readonly("samples", &Pod5SignalCacheBatch::samples); + py::class_>( + m, "Pod5SignalCacheBatch") + .def_property_readonly("batch_index", &Pod5SignalCacheBatch::batch_index) + .def_property_readonly("sample_count", &Pod5SignalCacheBatch::sample_count) + .def_property_readonly("samples", &Pod5SignalCacheBatch::samples); py::class_(m, "Pod5FileReader") - .def("get_file_run_info_table_location", - &Pod5FileReaderPtr::get_file_run_info_table_location) - .def("get_file_read_table_location", &Pod5FileReaderPtr::get_file_read_table_location) - .def("get_file_signal_table_location", - &Pod5FileReaderPtr::get_file_signal_table_location) - .def("get_file_version_pre_migration", - &Pod5FileReaderPtr::get_file_version_pre_migration) - .def("plan_traversal", &Pod5FileReaderPtr::plan_traversal) - .def("batch_get_signal", &Pod5FileReaderPtr::batch_get_signal) - .def("batch_get_signal_selection", &Pod5FileReaderPtr::batch_get_signal_selection) - .def("batch_get_signal_batches", &Pod5FileReaderPtr::batch_get_signal_batches) - .def("close", &Pod5FileReaderPtr::close); + .def( + "get_file_run_info_table_location", + &Pod5FileReaderPtr::get_file_run_info_table_location) + .def("get_file_read_table_location", &Pod5FileReaderPtr::get_file_read_table_location) + .def("get_file_signal_table_location", &Pod5FileReaderPtr::get_file_signal_table_location) + .def("get_file_version_pre_migration", &Pod5FileReaderPtr::get_file_version_pre_migration) + .def("plan_traversal", &Pod5FileReaderPtr::plan_traversal) + .def("batch_get_signal", &Pod5FileReaderPtr::batch_get_signal) + .def("batch_get_signal_selection", &Pod5FileReaderPtr::batch_get_signal_selection) + .def("batch_get_signal_batches", &Pod5FileReaderPtr::batch_get_signal_batches) + .def("close", &Pod5FileReaderPtr::close); // Errors API m.def("get_error_string", &pod5_get_error_string, "Get the most recent error as a string"); // Creating files - m.def("create_file", &create_file, "Create a POD5 file for writing", py::arg("filename"), - py::arg("writer_name"), py::arg("options") = nullptr); + m.def( + "create_file", + &create_file, + "Create a POD5 file for writing", + py::arg("filename"), + py::arg("writer_name"), + py::arg("options") = nullptr); // Opening files m.def("open_file", &open_file, "Open a POD5 file for reading"); - m.def("update_file", &write_updated_file_to_dest, - "Update a POD5 file to the latest writer format"); + m.def( + "update_file", + &write_updated_file_to_dest, + "Update a POD5 file to the latest writer format"); // Signal API m.def("decompress_signal", &decompress_signal_wrapper, "Decompress a numpy array of signal"); @@ -83,21 +100,23 @@ PYBIND11_MODULE(pod5_format_pybind, m) { py::class_>(m, "Pod5RepackerOutput"); py::class_>(m, "Repacker") - .def(py::init<>()) - .def("add_output", &Pod5Repacker::add_output) - .def("add_all_reads_to_output", &Pod5Repacker::add_all_reads_to_output) - .def("add_selected_reads_to_output", &Pod5Repacker::add_selected_reads_to_output) - .def("finish", &Pod5Repacker::finish) - .def_property_readonly("is_complete", &Pod5Repacker::is_complete) - .def_property_readonly("pending_batch_writes", &Pod5Repacker::pending_batch_writes) - .def_property_readonly("reads_completed", &Pod5Repacker::reads_completed) - .def_property_readonly("reads_sample_bytes_completed", - &Pod5Repacker::reads_sample_bytes_completed) - .def_property_readonly("batches_requested", &Pod5Repacker::batches_requested) - .def_property_readonly("batches_completed", &Pod5Repacker::batches_completed); + .def(py::init<>()) + .def("add_output", &Pod5Repacker::add_output) + .def("add_all_reads_to_output", &Pod5Repacker::add_all_reads_to_output) + .def("add_selected_reads_to_output", &Pod5Repacker::add_selected_reads_to_output) + .def("finish", &Pod5Repacker::finish) + .def_property_readonly("is_complete", &Pod5Repacker::is_complete) + .def_property_readonly("pending_batch_writes", &Pod5Repacker::pending_batch_writes) + .def_property_readonly("reads_completed", &Pod5Repacker::reads_completed) + .def_property_readonly( + "reads_sample_bytes_completed", &Pod5Repacker::reads_sample_bytes_completed) + .def_property_readonly("batches_requested", &Pod5Repacker::batches_requested) + .def_property_readonly("batches_completed", &Pod5Repacker::batches_completed); // Util API - m.def("load_read_id_iterable", &load_read_id_iterable, - "Load an iterable of read ids into a numpy array of data"); + m.def( + "load_read_id_iterable", + &load_read_id_iterable, + "Load an iterable of read ids into a numpy array of data"); m.def("format_read_id_to_str", &format_read_id_to_str, "Format an array of read ids to string"); } diff --git a/c++/pod5_format_pybind/repacker.h b/c++/pod5_format_pybind/repacker.h index e56cc6c..31d0c61 100644 --- a/c++/pod5_format_pybind/repacker.h +++ b/c++/pod5_format_pybind/repacker.h @@ -16,7 +16,8 @@ class Pod5Repacker; struct pair_hasher { template - std::size_t operator()(const std::pair& pair) const { + std::size_t operator()(std::pair const & pair) const + { return std::hash{}(pair.first) ^ std::hash{}(pair.second); } }; @@ -37,21 +38,26 @@ using WriteIndex = std::uint64_t; class Pod5RepackerOutput; struct AddReadBatchToOutput { - AddReadBatchToOutput(std::shared_ptr output_, - WriteIndex write_index_, - Pod5FileReaderPtr input_, - std::size_t read_batch_index_) - : output(output_), - write_index(write_index_), - input(input_), - read_batch_index(read_batch_index_) {} - - AddReadBatchToOutput(std::shared_ptr output_, - WriteIndex write_index_, - Pod5FileReaderPtr input_, - std::size_t read_batch_index_, - std::vector&& selected_rows_) - : AddReadBatchToOutput(output_, write_index_, input_, read_batch_index_) { + AddReadBatchToOutput( + std::shared_ptr output_, + WriteIndex write_index_, + Pod5FileReaderPtr input_, + std::size_t read_batch_index_) + : output(output_) + , write_index(write_index_) + , input(input_) + , read_batch_index(read_batch_index_) + { + } + + AddReadBatchToOutput( + std::shared_ptr output_, + WriteIndex write_index_, + Pod5FileReaderPtr input_, + std::size_t read_batch_index_, + std::vector && selected_rows_) + : AddReadBatchToOutput(output_, write_index_, input_, read_batch_index_) + { selected_rows = std::move(selected_rows_); } @@ -64,14 +70,21 @@ struct AddReadBatchToOutput { class Pod5ReadBatch { public: - Pod5ReadBatch(pod5::ReadTableRecordBatch&& batch, - std::shared_ptr const& file, - ReadSignalBatch&& read_signal) - : m_batch(std::move(batch)), m_file(file), m_read_signal(std::move(read_signal)) {} + Pod5ReadBatch( + pod5::ReadTableRecordBatch && batch, + std::shared_ptr const & file, + ReadSignalBatch && read_signal) + : m_batch(std::move(batch)) + , m_file(file) + , m_read_signal(std::move(read_signal)) + { + } + + pod5::ReadTableRecordBatch & batch() { return m_batch; } + + std::shared_ptr & file() { return m_file; } - pod5::ReadTableRecordBatch& batch() { return m_batch; } - std::shared_ptr& file() { return m_file; } - ReadSignalBatch const& read_signal() const { return m_read_signal; } + ReadSignalBatch const & read_signal() const { return m_read_signal; } private: pod5::ReadTableRecordBatch m_batch; @@ -88,45 +101,55 @@ class Pod5RepackerOutput { std::function complete; }; - Pod5RepackerOutput(std::shared_ptr const& repacker, - boost::asio::io_context& context, - std::shared_ptr const& output_file) - : m_strand(context), - m_repacker(repacker), - m_output_file(output_file), - m_signal_type(output_file->signal_type()), - m_queued_reads(0), - m_pending_write_count(0), - m_reads_completed(0), - m_reads_sample_bytes_completed(0), - m_has_error(false) {} + Pod5RepackerOutput( + std::shared_ptr const & repacker, + boost::asio::io_context & context, + std::shared_ptr const & output_file) + : m_strand(context) + , m_repacker(repacker) + , m_output_file(output_file) + , m_signal_type(output_file->signal_type()) + , m_queued_reads(0) + , m_pending_write_count(0) + , m_reads_completed(0) + , m_reads_sample_bytes_completed(0) + , m_has_error(false) + { + } std::shared_ptr repacker() const { return m_repacker; } + std::shared_ptr output_file() const { return m_output_file; } + pod5::SignalType signal_type() const { return m_signal_type; } template - void batch_write(WriteIndex index, - std::vector&& batch_rows, - std::shared_ptr const& batch, - CompletionHandler complete) { + void batch_write( + WriteIndex index, + std::vector && batch_rows, + std::shared_ptr const & batch, + CompletionHandler complete) + { m_pending_write_count += 1; m_strand.post([this, index, batch, complete, batch_rows = std::move(batch_rows)] { m_pending_writes.push_back({index, batch, std::move(batch_rows), complete}); - std::sort(m_pending_writes.begin(), m_pending_writes.end(), - [](auto const& a, auto const& b) { return a.index < b.index; }); + std::sort( + m_pending_writes.begin(), + m_pending_writes.end(), + [](auto const & a, auto const & b) { return a.index < b.index; }); try_write_next_batch(); }); } - void try_write_next_batch() { + void try_write_next_batch() + { if (m_pending_writes.empty()) { return; } - auto& next_batch = m_pending_writes.front(); + auto & next_batch = m_pending_writes.front(); if (next_batch.index == m_next_write_write_index) { auto result = write_next_batch(next_batch.batch, next_batch.selected_rows); next_batch.complete(); @@ -145,76 +168,82 @@ class Pod5RepackerOutput { } } - arrow::Status write_next_batch(std::shared_ptr const& batch, - std::vector const& selected_row_indices) { + arrow::Status write_next_batch( + std::shared_ptr const & batch, + std::vector const & selected_row_indices) + { POD5_TRACE_FUNCTION(); // Move signal between the two locations: - auto const& source_read_table_batch = batch->batch(); - auto const& source_file = batch->file(); + auto const & source_read_table_batch = batch->batch(); + auto const & source_file = batch->file(); m_reads_completed += source_read_table_batch.num_rows(); ARROW_ASSIGN_OR_RAISE(auto columns, source_read_table_batch.columns()); auto source_reads_pore_type_column = - std::static_pointer_cast(columns.pore_type->indices()); + std::static_pointer_cast(columns.pore_type->indices()); auto source_reads_end_reason_column = - std::static_pointer_cast(columns.end_reason->indices()); + std::static_pointer_cast(columns.end_reason->indices()); auto source_reads_run_info_column = - std::static_pointer_cast(columns.run_info->indices()); + std::static_pointer_cast(columns.run_info->indices()); - auto const& loaded_signal = batch->read_signal(); + auto const & loaded_signal = batch->read_signal(); assert(loaded_signal.data.size() == selected_row_indices.size()); for (std::size_t batch_row_index = 0; batch_row_index < selected_row_indices.size(); - ++batch_row_index) { + ++batch_row_index) + { auto batch_row = selected_row_indices[batch_row_index]; // Find the read params - auto const& read_id = columns.read_id->Value(batch_row); - auto const& read_number = columns.read_number->Value(batch_row); - auto const& start_sample = columns.start_sample->Value(batch_row); - auto const& channel = columns.channel->Value(batch_row); - auto const& well = columns.well->Value(batch_row); - auto const& calibration_offset = columns.calibration_offset->Value(batch_row); - auto const& calibration_scale = columns.calibration_scale->Value(batch_row); - auto const& median_before = columns.median_before->Value(batch_row); - auto const& end_reason_forced = columns.end_reason_forced->Value(batch_row); - auto const& num_minknow_events = columns.num_minknow_events->Value(batch_row); - auto const& tracked_scaling_scale = columns.tracked_scaling_scale->Value(batch_row); - auto const& tracked_scaling_shift = columns.tracked_scaling_shift->Value(batch_row); - auto const& predicted_scaling_scale = columns.predicted_scaling_scale->Value(batch_row); - auto const& predicted_scaling_shift = columns.predicted_scaling_shift->Value(batch_row); - auto const& num_reads_since_mux_change = - columns.num_reads_since_mux_change->Value(batch_row); - auto const& time_since_mux_change = columns.time_since_mux_change->Value(batch_row); - - auto const& pore_type_index = source_reads_pore_type_column->Value(batch_row); - auto const& end_reason_index = source_reads_end_reason_column->Value(batch_row); - auto const& run_info_index = source_reads_run_info_column->Value(batch_row); + auto const & read_id = columns.read_id->Value(batch_row); + auto const & read_number = columns.read_number->Value(batch_row); + auto const & start_sample = columns.start_sample->Value(batch_row); + auto const & channel = columns.channel->Value(batch_row); + auto const & well = columns.well->Value(batch_row); + auto const & calibration_offset = columns.calibration_offset->Value(batch_row); + auto const & calibration_scale = columns.calibration_scale->Value(batch_row); + auto const & median_before = columns.median_before->Value(batch_row); + auto const & end_reason_forced = columns.end_reason_forced->Value(batch_row); + auto const & num_minknow_events = columns.num_minknow_events->Value(batch_row); + auto const & tracked_scaling_scale = columns.tracked_scaling_scale->Value(batch_row); + auto const & tracked_scaling_shift = columns.tracked_scaling_shift->Value(batch_row); + auto const & predicted_scaling_scale = + columns.predicted_scaling_scale->Value(batch_row); + auto const & predicted_scaling_shift = + columns.predicted_scaling_shift->Value(batch_row); + auto const & num_reads_since_mux_change = + columns.num_reads_since_mux_change->Value(batch_row); + auto const & time_since_mux_change = columns.time_since_mux_change->Value(batch_row); + + auto const & pore_type_index = source_reads_pore_type_column->Value(batch_row); + auto const & end_reason_index = source_reads_end_reason_column->Value(batch_row); + auto const & run_info_index = source_reads_run_info_column->Value(batch_row); std::vector signal_rows; - auto const& read_signal = loaded_signal.data[batch_row_index]; + auto const & read_signal = loaded_signal.data[batch_row_index]; std::uint64_t total_sample_count = 0; // Write each compressed row to the dest file, and store its rows: for (std::size_t i = 0; i < read_signal.signal_data.size(); ++i) { - auto const& sample_count = read_signal.sample_counts[i]; - auto const& signal_buffer = read_signal.signal_data[i]; + auto const & sample_count = read_signal.sample_counts[i]; + auto const & signal_buffer = read_signal.signal_data[i]; auto const signal_span = - gsl::make_span(signal_buffer->data(), signal_buffer->size()); + gsl::make_span(signal_buffer->data(), signal_buffer->size()); if (loaded_signal.signal_type == m_output_file->signal_type()) { - ARROW_ASSIGN_OR_RAISE(auto signal_row, - m_output_file->add_pre_compressed_signal( - read_id, signal_span, sample_count)); + ARROW_ASSIGN_OR_RAISE( + auto signal_row, + m_output_file->add_pre_compressed_signal( + read_id, signal_span, sample_count)); signal_rows.push_back(signal_row); } else { ARROW_ASSIGN_OR_RAISE( - auto new_signal_rows, - m_output_file->add_signal(read_id, - signal_span.as_span())); - signal_rows.insert(signal_rows.end(), new_signal_rows.begin(), - new_signal_rows.end()); + auto new_signal_rows, + m_output_file->add_signal( + read_id, signal_span.as_span())); + signal_rows.insert( + signal_rows.end(), new_signal_rows.begin(), new_signal_rows.end()); } total_sample_count += sample_count; @@ -222,24 +251,38 @@ class Pod5RepackerOutput { } ARROW_ASSIGN_OR_RAISE( - auto dest_pore_index, - find_pore_index(source_file, source_read_table_batch, pore_type_index)); + auto dest_pore_index, + find_pore_index(source_file, source_read_table_batch, pore_type_index)); ARROW_ASSIGN_OR_RAISE( - auto dest_end_reason_index, - find_end_reason_index(source_file, source_read_table_batch, end_reason_index)); + auto dest_end_reason_index, + find_end_reason_index(source_file, source_read_table_batch, end_reason_index)); ARROW_ASSIGN_OR_RAISE( - auto dest_run_info_index, - find_run_info_index(source_file, source_read_table_batch, run_info_index)); + auto dest_run_info_index, + find_run_info_index(source_file, source_read_table_batch, run_info_index)); ARROW_RETURN_NOT_OK(m_output_file->add_complete_read( - pod5::ReadData(read_id, read_number, start_sample, channel, well, - dest_pore_index, calibration_offset, calibration_scale, - median_before, dest_end_reason_index, end_reason_forced, - dest_run_info_index, num_minknow_events, tracked_scaling_scale, - tracked_scaling_shift, predicted_scaling_scale, - predicted_scaling_shift, num_reads_since_mux_change, - time_since_mux_change), - signal_rows, total_sample_count)); + pod5::ReadData( + read_id, + read_number, + start_sample, + channel, + well, + dest_pore_index, + calibration_offset, + calibration_scale, + median_before, + dest_end_reason_index, + end_reason_forced, + dest_run_info_index, + num_minknow_events, + tracked_scaling_scale, + tracked_scaling_shift, + predicted_scaling_scale, + predicted_scaling_shift, + num_reads_since_mux_change, + time_since_mux_change), + signal_rows, + total_sample_count)); } return arrow::Status::OK(); @@ -247,9 +290,10 @@ class Pod5RepackerOutput { // Find or create a pore index in the output file - expects to run on strand. arrow::Result find_pore_index( - std::shared_ptr const& source_file, - pod5::ReadTableRecordBatch const& source_batch, - pod5::PoreDictionaryIndex source_index) { + std::shared_ptr const & source_file, + pod5::ReadTableRecordBatch const & source_batch, + pod5::PoreDictionaryIndex source_index) + { auto const key = std::make_pair(source_file, source_index); auto const it = m_pore_indexes.find(key); if (it != m_pore_indexes.end()) { @@ -264,9 +308,10 @@ class Pod5RepackerOutput { // Find or create a end reason index in the output file - expects to run on strand. arrow::Result find_end_reason_index( - std::shared_ptr const& source_file, - pod5::ReadTableRecordBatch const& source_batch, - pod5::EndReasonDictionaryIndex source_index) { + std::shared_ptr const & source_file, + pod5::ReadTableRecordBatch const & source_batch, + pod5::EndReasonDictionaryIndex source_index) + { auto const key = std::make_pair(source_file, source_index); auto const it = m_end_reason_indexes.find(key); if (it != m_end_reason_indexes.end()) { @@ -274,17 +319,18 @@ class Pod5RepackerOutput { } ARROW_ASSIGN_OR_RAISE(auto source_data, source_batch.get_end_reason(source_index)); - ARROW_ASSIGN_OR_RAISE(auto const new_index, - m_output_file->lookup_end_reason(source_data.first)); + ARROW_ASSIGN_OR_RAISE( + auto const new_index, m_output_file->lookup_end_reason(source_data.first)); m_end_reason_indexes[key] = new_index; return new_index; } // Find or create a run_info index in the output file - expects to run on strand. arrow::Result find_run_info_index( - std::shared_ptr const& source_file, - pod5::ReadTableRecordBatch const& source_batch, - pod5::RunInfoDictionaryIndex source_index) { + std::shared_ptr const & source_file, + pod5::ReadTableRecordBatch const & source_batch, + pod5::RunInfoDictionaryIndex source_index) + { auto const key = std::make_pair(source_file, source_index); auto const it = m_run_info_indexes.find(key); if (it != m_run_info_indexes.end()) { @@ -305,16 +351,18 @@ class Pod5RepackerOutput { // Add reads which have been pushed onto a thread for execution void add_queued_reads(std::size_t additional_reads) { m_queued_reads += additional_reads; } - void add_pending_reads(std::vector&& reads) { + void add_pending_reads(std::vector && reads) + { auto pending_batch_reads = m_pending_batch_reads.synchronize(); - for (auto& read : reads) { + for (auto & read : reads) { pending_batch_reads->emplace_back(std::move(read)); } } // Pop a read from the pending reads, and remove one queued read. - boost::optional pop_pending_read() { + boost::optional pop_pending_read() + { assert(m_queued_reads > 0); boost::optional task; auto pending_batch_reads = m_pending_batch_reads.synchronize(); @@ -329,19 +377,26 @@ class Pod5RepackerOutput { } std::size_t pending_unqueued_reads() { return m_pending_batch_reads->size(); } + std::size_t queued_reads() { return m_queued_reads.load(); } + std::size_t pending_writes() { return m_pending_write_count.load(); } + std::size_t pending_actions() { return queued_reads() + pending_writes(); } + std::size_t reads_completed() { return m_reads_completed.load(); } + std::size_t reads_sample_bytes_completed() { return m_reads_sample_bytes_completed.load(); } - boost::asio::io_context::strand& strand() { return m_strand; } + boost::asio::io_context::strand & strand() { return m_strand; } + + arrow::Status const & error() { return *m_error; } - arrow::Status const& error() { return *m_error; } bool has_error() const { return m_has_error.load(); } private: - void set_error(arrow::Status const& error) { + void set_error(arrow::Status const & error) + { m_error = error; m_has_error = true; } @@ -372,10 +427,10 @@ class Pod5RepackerOutput { WriteIndex m_next_write_write_index = 0; template - using DictionaryLookup = - std::unordered_map, IndexType>, - IndexType, - pair_hasher>; + using DictionaryLookup = std::unordered_map< + std::pair, IndexType>, + IndexType, + pair_hasher>; DictionaryLookup m_pore_indexes; DictionaryLookup m_calibration_indexes; DictionaryLookup m_end_reason_indexes; @@ -384,13 +439,15 @@ class Pod5RepackerOutput { class Pod5Repacker : public std::enable_shared_from_this { public: - Pod5Repacker(std::size_t target_pending_writes = 20, - std::size_t worker_count = std::thread::hardware_concurrency()) - : m_target_pending_writes(target_pending_writes), - m_has_error(false), - m_batches_requested(0), - m_batches_completed(0), - m_work(boost::asio::make_work_guard(m_context)) { + Pod5Repacker( + std::size_t target_pending_writes = 20, + std::size_t worker_count = std::thread::hardware_concurrency()) + : m_target_pending_writes(target_pending_writes) + , m_has_error(false) + , m_batches_requested(0) + , m_batches_completed(0) + , m_work(boost::asio::make_work_guard(m_context)) + { m_workers.reserve(worker_count); for (std::size_t i = 0; i < worker_count; ++i) { m_workers.emplace_back([&] { m_context.run(); }); @@ -399,25 +456,28 @@ class Pod5Repacker : public std::enable_shared_from_this { ~Pod5Repacker() { finish(); } - boost::asio::io_context& io_context() { return m_context; } + boost::asio::io_context & io_context() { return m_context; } - void finish() { + void finish() + { m_work.reset(); - for (auto& worker : m_workers) { + for (auto & worker : m_workers) { worker.join(); } } - std::shared_ptr add_output( - std::shared_ptr const& output) { + std::shared_ptr add_output(std::shared_ptr const & output) + { auto repacker_output = - std::make_shared(shared_from_this(), m_context, output); + std::make_shared(shared_from_this(), m_context, output); m_outputs.push_back(repacker_output); return repacker_output; } - void add_all_reads_to_output(std::shared_ptr const& output, - Pod5FileReaderPtr const& input) { + void add_all_reads_to_output( + std::shared_ptr const & output, + Pod5FileReaderPtr const & input) + { if (output->repacker() != shared_from_this()) { throw std::runtime_error("Invalid repacker output passed, created by another repacker"); } @@ -437,11 +497,11 @@ class Pod5Repacker : public std::enable_shared_from_this { } void add_selected_reads_to_output( - std::shared_ptr const& output, - Pod5FileReaderPtr const& input, - py::array_t&& batch_counts, - py::array_t&& - all_batch_rows) { + std::shared_ptr const & output, + Pod5FileReaderPtr const & input, + py::array_t && batch_counts, + py::array_t && all_batch_rows) + { if (output->repacker() != shared_from_this()) { throw std::runtime_error("Invalid repacker output passed, created by another repacker"); } @@ -460,7 +520,7 @@ class Pod5Repacker : public std::enable_shared_from_this { for (std::size_t i = 0; i < batch_counts_span.size(); ++i) { std::vector batch_rows; auto const batch_rows_span = - all_batch_rows_span.subspan(current_start_point, batch_counts_span[i]); + all_batch_rows_span.subspan(current_start_point, batch_counts_span[i]); // If this batch has no selected if (batch_rows_span.empty()) { @@ -470,8 +530,8 @@ class Pod5Repacker : public std::enable_shared_from_this { batch_rows.insert(batch_rows.end(), batch_rows_span.begin(), batch_rows_span.end()); current_start_point += batch_counts_span[i]; - new_reads.emplace_back(output, output->get_next_write_index(), input, i, - std::move(batch_rows)); + new_reads.emplace_back( + output, output->get_next_write_index(), input, i, std::move(batch_rows)); added_reads += 1; } output->add_pending_reads(std::move(new_reads)); @@ -480,12 +540,13 @@ class Pod5Repacker : public std::enable_shared_from_this { post_do_batch_reads(output, m_target_pending_writes); } - bool is_complete() const { + bool is_complete() const + { if (m_has_error) { throw std::runtime_error(m_error->ToString()); } - for (auto const& output : m_outputs) { + for (auto const & output : m_outputs) { if (output->has_error()) { throw std::runtime_error(output->error().ToString()); } @@ -494,7 +555,7 @@ class Pod5Repacker : public std::enable_shared_from_this { } } - for (auto const& output : m_outputs) { + for (auto const & output : m_outputs) { if (output->pending_unqueued_reads() > 0) { return false; } @@ -502,34 +563,42 @@ class Pod5Repacker : public std::enable_shared_from_this { return true; } - std::size_t reads_sample_bytes_completed() const { + std::size_t reads_sample_bytes_completed() const + { std::size_t reads_sample_bytes_completed = 0; - for (auto const& output : m_outputs) { + for (auto const & output : m_outputs) { reads_sample_bytes_completed += output->reads_sample_bytes_completed(); } return reads_sample_bytes_completed; } - std::size_t reads_completed() const { + + std::size_t reads_completed() const + { std::size_t reads_completed = 0; - for (auto const& output : m_outputs) { + for (auto const & output : m_outputs) { reads_completed += output->reads_completed(); } return reads_completed; } - std::size_t pending_batch_writes() const { + + std::size_t pending_batch_writes() const + { std::size_t pending_batch_writes = 0; - for (auto const& output : m_outputs) { + for (auto const & output : m_outputs) { pending_batch_writes += output->pending_writes(); } return pending_batch_writes; } std::size_t batches_requested() const { return m_batches_requested.load(); } + std::size_t batches_completed() const { return m_batches_completed.load(); } private: - void post_do_batch_reads(std::shared_ptr const& output, - std::size_t force_count = 0) { + void post_do_batch_reads( + std::shared_ptr const & output, + std::size_t force_count = 0) + { assert(output); auto pending_actions = output->pending_actions(); @@ -538,7 +607,7 @@ class Pod5Repacker : public std::enable_shared_from_this { auto pending_write_target = force_count; if (pending_actions < output_target) { pending_write_target = - std::max(pending_write_target, m_target_pending_writes - pending_actions); + std::max(pending_write_target, m_target_pending_writes - pending_actions); } if (pending_write_target == 0) { @@ -559,29 +628,36 @@ class Pod5Repacker : public std::enable_shared_from_this { // Do the task: auto selected_rows = std::move(task->selected_rows); - auto batch = read_batch(task->input.reader, task->output->signal_type(), - selected_rows, task->read_batch_index); + auto batch = read_batch( + task->input.reader, + task->output->signal_type(), + selected_rows, + task->read_batch_index); if (!batch.ok()) { set_error(batch.status()); return; } - task->output->batch_write(task->write_index, std::move(selected_rows), *batch, - [this, output = task->output] { - m_batches_completed += 1; + task->output->batch_write( + task->write_index, + std::move(selected_rows), + *batch, + [this, output = task->output] { + m_batches_completed += 1; - // And post the next batch read now we are complete: - post_do_batch_reads(output); - }); + // And post the next batch read now we are complete: + post_do_batch_reads(output); + }); }); } } pod5::Result> read_batch( - std::shared_ptr const& source_file, - pod5::SignalType dest_signal_type, - std::vector& selected_rows, - std::size_t batch_index) { + std::shared_ptr const & source_file, + pod5::SignalType dest_signal_type, + std::vector & selected_rows, + std::size_t batch_index) + { POD5_TRACE_FUNCTION(); ARROW_ASSIGN_OR_RAISE(auto read_batch, source_file->read_read_record_batch(batch_index)); @@ -604,47 +680,50 @@ class Pod5Repacker : public std::enable_shared_from_this { for (auto const batch_row : selected_rows) { // Get the signal row data for the read: auto const signal_rows = std::static_pointer_cast( - source_reads_signal_column->value_slice(batch_row)); + source_reads_signal_column->value_slice(batch_row)); auto const signal_rows_span = - gsl::make_span(signal_rows->raw_values(), signal_rows->length()); + gsl::make_span(signal_rows->raw_values(), signal_rows->length()); ReadSignalBatch::ReadSignal row_signal; // If were using the same compression type in both files, just copy compressed: if (input_signal_type == dest_signal_type) { - ARROW_ASSIGN_OR_RAISE(row_signal.signal_data, - source_file->extract_samples_inplace( - signal_rows_span, row_signal.sample_counts)); + ARROW_ASSIGN_OR_RAISE( + row_signal.signal_data, + source_file->extract_samples_inplace( + signal_rows_span, row_signal.sample_counts)); } else { // Find the sample count of the complete read: - ARROW_ASSIGN_OR_RAISE(auto sample_count, - source_file->extract_sample_count(signal_rows_span)); + ARROW_ASSIGN_OR_RAISE( + auto sample_count, source_file->extract_sample_count(signal_rows_span)); row_signal.sample_counts.push_back(sample_count); // Read the samples: - ARROW_ASSIGN_OR_RAISE(std::shared_ptr buffer, - arrow::AllocateBuffer(sample_count * sizeof(std::int16_t))); + ARROW_ASSIGN_OR_RAISE( + std::shared_ptr buffer, + arrow::AllocateBuffer(sample_count * sizeof(std::int16_t))); row_signal.signal_data.push_back(buffer); - auto signal_buffer_span = gsl::make_span(buffer->mutable_data(), buffer->size()) - .as_span(); + auto signal_buffer_span = + gsl::make_span(buffer->mutable_data(), buffer->size()).as_span(); ARROW_RETURN_NOT_OK( - source_file->extract_samples(signal_rows_span, signal_buffer_span)); + source_file->extract_samples(signal_rows_span, signal_buffer_span)); } read_signal.preload_sum += cache_signal(row_signal.signal_data); read_signal.data.emplace_back(std::move(row_signal)); } - return std::make_shared(std::move(read_batch), source_file, - std::move(read_signal)); + return std::make_shared( + std::move(read_batch), source_file, std::move(read_signal)); } - std::int64_t cache_signal(std::vector> const& row) { + std::int64_t cache_signal(std::vector> const & row) + { POD5_TRACE_FUNCTION(); std::size_t step_size = 512; std::int64_t sum = 0; - for (auto const& section : row) { + for (auto const & section : row) { for (std::size_t i = 0; i < section->size(); i += step_size) { sum += section->data()[i]; } @@ -652,7 +731,8 @@ class Pod5Repacker : public std::enable_shared_from_this { return sum; } - void set_error(arrow::Status const& error) { + void set_error(arrow::Status const & error) + { m_error = error; m_has_error = true; } diff --git a/c++/pod5_format_pybind/utils.h b/c++/pod5_format_pybind/utils.h index 58dfb38..39c7594 100644 --- a/c++/pod5_format_pybind/utils.h +++ b/c++/pod5_format_pybind/utils.h @@ -2,12 +2,14 @@ #include "pod5_format/result.h" -inline void raise_error(arrow::Status const& status) { +inline void raise_error(arrow::Status const & status) +{ throw std::runtime_error(status.ToString()); } template -inline void raise_error(arrow::Result const& result) { +inline void raise_error(arrow::Result const & result) +{ throw std::runtime_error(result.status().ToString()); } @@ -17,24 +19,26 @@ inline void raise_error(arrow::Result const& result) { } #define POD5_PYTHON_ASSIGN_OR_RAISE_IMPL(result_name, lhs, rexpr) \ - auto&& result_name = (rexpr); \ + auto && result_name = (rexpr); \ if (!(result_name).ok()) { \ raise_error(result_name); \ } \ lhs = std::move(result_name).ValueUnsafe(); -#define POD5_PYTHON_ASSIGN_OR_RAISE(lhs, rexpr) \ - POD5_PYTHON_ASSIGN_OR_RAISE_IMPL(ARROW_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), \ - lhs, rexpr); +#define POD5_PYTHON_ASSIGN_OR_RAISE(lhs, rexpr) \ + POD5_PYTHON_ASSIGN_OR_RAISE_IMPL( \ + ARROW_ASSIGN_OR_RAISE_NAME(_error_or_value, __COUNTER__), lhs, rexpr); -inline void throw_on_error(pod5::Status const& s) { +inline void throw_on_error(pod5::Status const & s) +{ if (!s.ok()) { throw std::runtime_error(s.ToString()); } } template -inline T throw_on_error(pod5::Result const& s) { +inline T throw_on_error(pod5::Result const & s) +{ if (!s.ok()) { throw std::runtime_error(s.status().ToString()); } diff --git a/c++/test/c_api_tests.cpp b/c++/test/c_api_tests.cpp index d2c1405..eccaad3 100644 --- a/c++/test/c_api_tests.cpp +++ b/c++/test/c_api_tests.cpp @@ -13,8 +13,9 @@ #include #include -SCENARIO("C API") { - static constexpr char const* filename = "./foo_c_api.pod5"; +SCENARIO("C API") +{ + static constexpr char const * filename = "./foo_c_api.pod5"; pod5_init(); auto fin = gsl::finally([] { pod5_terminate(); }); @@ -63,20 +64,41 @@ SCENARIO("C API") { CHECK(pod5_add_pore(&pore_type_id, file, "pore_type") == POD5_OK); CHECK(pore_type_id == 0); - std::vector context_tags_keys{"thing", "foo"}; - std::vector context_tags_values{"thing_val", "foo_val"}; - std::vector tracking_id_keys{"baz", "other"}; - std::vector tracking_id_values{"baz_val", "other_val"}; + std::vector context_tags_keys{"thing", "foo"}; + std::vector context_tags_values{"thing_val", "foo_val"}; + std::vector tracking_id_keys{"baz", "other"}; + std::vector tracking_id_values{"baz_val", "other_val"}; std::int16_t run_info_id = -1; - CHECK(pod5_add_run_info(&run_info_id, file, "acquisition_id", 15400, adc_max, adc_min, - context_tags_keys.size(), context_tags_keys.data(), - context_tags_values.data(), "experiment_name", "flow_cell_id", - "flow_cell_product_code", "protocol_name", "protocol_run_id", - 200000, "sample_id", 4000, "sequencing_kit", "sequencer_position", - "sequencer_position_type", "software", "system_name", "system_type", - tracking_id_keys.size(), tracking_id_keys.data(), - tracking_id_values.data()) == POD5_OK); + CHECK( + pod5_add_run_info( + &run_info_id, + file, + "acquisition_id", + 15400, + adc_max, + adc_min, + context_tags_keys.size(), + context_tags_keys.data(), + context_tags_values.data(), + "experiment_name", + "flow_cell_id", + "flow_cell_product_code", + "protocol_name", + "protocol_run_id", + 200000, + "sample_id", + 4000, + "sequencing_kit", + "sequencer_position", + "sequencer_position_type", + "software", + "system_name", + "system_type", + tracking_id_keys.size(), + tracking_id_keys.data(), + tracking_id_values.data()) + == POD5_OK); CHECK(run_info_id == 0); std::uint32_t read_number = 12; @@ -86,54 +108,65 @@ SCENARIO("C API") { std::uint8_t well = 4; pod5_end_reason_t end_reason = POD5_END_REASON_MUX_CHANGE; uint8_t end_reason_forced = false; - auto read_id_array = (read_id_t const*)input_read_id.begin(); - - ReadBatchRowInfoArrayV3 row_data{read_id_array, - &read_number, - &start_sample, - &median_before, - &channel, - &well, - &pore_type_id, - &calibration_offset, - &calibration_scale, - &end_reason, - &end_reason_forced, - &run_info_id, - &num_minknow_events, - &tracked_scale, - &tracked_shift, - &predicted_scale, - &predicted_shift, - &num_reads_since_mux_change, - &time_since_mux_change}; + auto read_id_array = (read_id_t const *)input_read_id.begin(); + + ReadBatchRowInfoArrayV3 row_data{ + read_id_array, + &read_number, + &start_sample, + &median_before, + &channel, + &well, + &pore_type_id, + &calibration_offset, + &calibration_scale, + &end_reason, + &end_reason_forced, + &run_info_id, + &num_minknow_events, + &tracked_scale, + &tracked_shift, + &predicted_scale, + &predicted_shift, + &num_reads_since_mux_change, + &time_since_mux_change}; { - std::int16_t const* signal_arr[] = {signal_1.data()}; + std::int16_t const * signal_arr[] = {signal_1.data()}; std::uint32_t signal_size[] = {(std::uint32_t)signal_1.size()}; - CHECK(pod5_add_reads_data(file, 1, READ_BATCH_ROW_INFO_VERSION_3, &row_data, signal_arr, - signal_size) == POD5_OK); + CHECK( + pod5_add_reads_data( + file, 1, READ_BATCH_ROW_INFO_VERSION_3, &row_data, signal_arr, signal_size) + == POD5_OK); read_count += 1; } { auto compressed_read_max_size = pod5_vbz_compressed_signal_max_size(signal_2.size()); std::vector compressed_signal(compressed_read_max_size); - char const* compressed_data[] = {compressed_signal.data()}; - char const** compressed_data_ptr = compressed_data; + char const * compressed_data[] = {compressed_signal.data()}; + char const ** compressed_data_ptr = compressed_data; std::size_t compressed_size[] = {compressed_signal.size()}; - std::size_t const* compressed_size_ptr = compressed_size; + std::size_t const * compressed_size_ptr = compressed_size; std::uint32_t signal_size[] = {(std::uint32_t)signal_2.size()}; - std::uint32_t const* signal_size_ptr = signal_size; - pod5_vbz_compress_signal(signal_2.data(), signal_2.size(), compressed_signal.data(), - compressed_size); + std::uint32_t const * signal_size_ptr = signal_size; + pod5_vbz_compress_signal( + signal_2.data(), signal_2.size(), compressed_signal.data(), compressed_size); std::size_t signal_counts = 1; - CHECK(pod5_add_reads_data_pre_compressed( - file, 1, READ_BATCH_ROW_INFO_VERSION_3, &row_data, &compressed_data_ptr, - &compressed_size_ptr, &signal_size_ptr, &signal_counts) == POD5_OK); + CHECK( + pod5_add_reads_data_pre_compressed( + file, + 1, + READ_BATCH_ROW_INFO_VERSION_3, + &row_data, + &compressed_data_ptr, + &compressed_size_ptr, + &signal_size_ptr, + &signal_counts) + == POD5_OK); read_count += 1; } @@ -157,9 +190,10 @@ SCENARIO("C API") { { auto reader = pod5::open_file_reader(filename); boost::uuids::uuid file_identifier; - std::copy(file_info.file_identifier, - file_info.file_identifier + sizeof(file_info.file_identifier), - file_identifier.begin()); + std::copy( + file_info.file_identifier, + file_info.file_identifier + sizeof(file_info.file_identifier), + file_identifier.begin()); CHECK(file_identifier == (*reader)->schema_metadata().file_identifier); } @@ -167,7 +201,7 @@ SCENARIO("C API") { CHECK(pod5_get_read_batch_count(&batch_count, file) == POD5_OK); REQUIRE(batch_count == 1); - Pod5ReadRecordBatch* batch_0 = nullptr; + Pod5ReadRecordBatch * batch_0 = nullptr; CHECK(pod5_get_read_batch(&batch_0, file, 0) == POD5_OK); REQUIRE(!!batch_0); @@ -177,21 +211,26 @@ SCENARIO("C API") { signal = signal_2; } - static_assert(std::is_same::value, - "Update this if new structs added"); + static_assert( + std::is_same::value, + "Update this if new structs added"); ReadBatchRowInfoV3 v3_struct; uint16_t input_version = 0; - CHECK(pod5_get_read_batch_row_info_data(batch_0, row, READ_BATCH_ROW_INFO_VERSION, - &v3_struct, &input_version) == POD5_OK); + CHECK( + pod5_get_read_batch_row_info_data( + batch_0, row, READ_BATCH_ROW_INFO_VERSION, &v3_struct, &input_version) + == POD5_OK); CHECK(input_version == 3); std::string formatted_uuid(36, '\0'); CHECK(pod5_format_read_id(v3_struct.read_id, &formatted_uuid[0]) == POD5_OK); - CHECK(formatted_uuid.size() == - boost::uuids::to_string(*(boost::uuids::uuid*)v3_struct.read_id).size()); - CHECK(formatted_uuid == - boost::uuids::to_string(*(boost::uuids::uuid*)v3_struct.read_id)); + CHECK( + formatted_uuid.size() + == boost::uuids::to_string(*(boost::uuids::uuid *)v3_struct.read_id).size()); + CHECK( + formatted_uuid + == boost::uuids::to_string(*(boost::uuids::uuid *)v3_struct.read_id)); CHECK(v3_struct.read_number == 12); CHECK(v3_struct.start_sample == 10245); @@ -215,57 +254,72 @@ SCENARIO("C API") { CHECK(v3_struct.num_samples == signal.size()); std::vector signal_row_indices(v3_struct.signal_row_count); - CHECK(pod5_get_signal_row_indices(batch_0, row, signal_row_indices.size(), - signal_row_indices.data()) == POD5_OK); - - std::vector signal_row_info(v3_struct.signal_row_count); - CHECK(pod5_get_signal_row_info(file, signal_row_indices.size(), - signal_row_indices.data(), - signal_row_info.data()) == POD5_OK); + CHECK( + pod5_get_signal_row_indices( + batch_0, row, signal_row_indices.size(), signal_row_indices.data()) + == POD5_OK); + + std::vector signal_row_info(v3_struct.signal_row_count); + CHECK( + pod5_get_signal_row_info( + file, + signal_row_indices.size(), + signal_row_indices.data(), + signal_row_info.data()) + == POD5_OK); std::vector read_signal(signal_row_info.front()->stored_sample_count); REQUIRE(signal_row_info.front()->stored_sample_count == signal.size()); - CHECK(pod5_get_signal(file, signal_row_info.front(), - signal_row_info.front()->stored_sample_count, - read_signal.data()) == POD5_OK); + CHECK( + pod5_get_signal( + file, + signal_row_info.front(), + signal_row_info.front()->stored_sample_count, + read_signal.data()) + == POD5_OK); CHECK(read_signal == signal); std::size_t sample_count = 0; - CHECK(pod5_get_read_complete_sample_count(file, batch_0, row, &sample_count) == - POD5_OK); + CHECK( + pod5_get_read_complete_sample_count(file, batch_0, row, &sample_count) == POD5_OK); CHECK(sample_count == signal_row_info.front()->stored_sample_count); - CHECK(pod5_get_read_complete_signal(file, batch_0, row, sample_count, - read_signal.data()) == POD5_OK); + CHECK( + pod5_get_read_complete_signal(file, batch_0, row, sample_count, read_signal.data()) + == POD5_OK); CHECK(read_signal == signal); - CHECK(pod5_free_signal_row_info(signal_row_indices.size(), signal_row_info.data()) == - POD5_OK); + CHECK( + pod5_free_signal_row_info(signal_row_indices.size(), signal_row_info.data()) + == POD5_OK); std::array char_buffer{}; std::size_t returned_size = char_buffer.size(); - CHECK(pod5_get_pore_type(batch_0, v3_struct.pore_type, char_buffer.data(), - &returned_size) == POD5_OK); + CHECK( + pod5_get_pore_type(batch_0, v3_struct.pore_type, char_buffer.data(), &returned_size) + == POD5_OK); std::string expected_pore_type{"pore_type"}; CHECK(returned_size == expected_pore_type.size() + 1); CHECK(std::string{char_buffer.data()} == expected_pore_type); returned_size = char_buffer.size(); pod5_end_reason end_reason = POD5_END_REASON_UNKNOWN; - CHECK(pod5_get_end_reason(batch_0, v3_struct.end_reason, &end_reason, - char_buffer.data(), &returned_size) == POD5_OK); + CHECK( + pod5_get_end_reason( + batch_0, v3_struct.end_reason, &end_reason, char_buffer.data(), &returned_size) + == POD5_OK); std::string expected_end_reason{"mux_change"}; CHECK(returned_size == expected_end_reason.size() + 1); CHECK(end_reason == POD5_END_REASON_MUX_CHANGE); CHECK(std::string{char_buffer.data()} == expected_end_reason); CalibrationExtraData calibration_extra_data{}; - CHECK(pod5_get_calibration_extra_info(batch_0, row, &calibration_extra_data) == - POD5_OK); + CHECK( + pod5_get_calibration_extra_info(batch_0, row, &calibration_extra_data) == POD5_OK); CHECK(calibration_extra_data.digitisation == adc_max - adc_min + 1); CHECK(calibration_extra_data.range == 8192 * calibration_scale); } - RunInfoDictData* run_info_data_out = nullptr; + RunInfoDictData * run_info_data_out = nullptr; CHECK(pod5_get_run_info(batch_0, 0, &run_info_data_out) == POD5_OK); REQUIRE(!!run_info_data_out); CHECK(run_info_data_out->tracking_id.size == 2); diff --git a/c++/test/file_reader_writer_tests.cpp b/c++/test/file_reader_writer_tests.cpp index 90d3373..a21a965 100644 --- a/c++/test/file_reader_writer_tests.cpp +++ b/c++/test/file_reader_writer_tests.cpp @@ -18,8 +18,9 @@ #include #include -void run_file_reader_writer_tests() { - static constexpr char const* file = "./foo.pod5"; +void run_file_reader_writer_tests() +{ + static constexpr char const * file = "./foo.pod5"; REQUIRE_ARROW_STATUS_OK(remove_file_if_exists(file)); (void)pod5::register_extension_types(); auto fin = gsl::finally([] { (void)pod5::unregister_extension_types(); }); @@ -64,12 +65,26 @@ void run_file_reader_writer_tests() { for (std::size_t i = 0; i < 10; ++i) { CHECK_ARROW_STATUS_OK((*writer)->add_complete_read( - {read_id_1, read_number, start_sample, channel, well, *pore_type, calib_offset, - calib_scale, median_before, *end_reason, end_reason_forced, *run_info, - num_minknow_events, tracked_scaling_scale, tracked_scaling_shift, - predicted_scaling_scale, predicted_scaling_shift, num_reads_since_mux_change, - time_since_mux_change}, - gsl::make_span(signal_1))); + {read_id_1, + read_number, + start_sample, + channel, + well, + *pore_type, + calib_offset, + calib_scale, + median_before, + *end_reason, + end_reason_forced, + *run_info, + num_minknow_events, + tracked_scaling_scale, + tracked_scaling_shift, + predicted_scaling_scale, + predicted_scaling_shift, + num_reads_since_mux_change, + time_since_mux_change}, + gsl::make_span(signal_1))); } } @@ -90,8 +105,7 @@ void run_file_reader_writer_tests() { auto columns = *read_batch->columns(); auto const run_info_dict_index = - std::dynamic_pointer_cast(columns.run_info->indices()) - ->Value(0); + std::dynamic_pointer_cast(columns.run_info->indices())->Value(0); CHECK(run_info_dict_index == 0); auto const run_info_id = read_batch->get_run_info(run_info_dict_index); CHECK(*run_info_id == run_info_data.acquisition_id); @@ -121,12 +135,15 @@ void run_file_reader_writer_tests() { CHECK(samples_array->Value(4) == 18'080); } - auto const samples_mode = GENERATE(pod5::AsyncSignalLoader::SamplesMode::NoSamples, - pod5::AsyncSignalLoader::SamplesMode::Samples); + auto const samples_mode = GENERATE( + pod5::AsyncSignalLoader::SamplesMode::NoSamples, + pod5::AsyncSignalLoader::SamplesMode::Samples); - pod5::AsyncSignalLoader async_no_samples_loader(*reader, samples_mode, - {}, // Read all the batches - {} // No specific rows within batches) + pod5::AsyncSignalLoader async_no_samples_loader( + *reader, + samples_mode, + {}, // Read all the batches + {} // No specific rows within batches) ); for (std::size_t i = 0; i < 10; ++i) { @@ -151,11 +168,12 @@ void run_file_reader_writer_tests() { SCENARIO("File Reader Writer Tests") { run_file_reader_writer_tests(); } -SCENARIO("Opening older files") { +SCENARIO("Opening older files") +{ (void)pod5::register_extension_types(); auto fin = gsl::finally([] { (void)pod5::unregister_extension_types(); }); - auto uuid_from_string = [](char const* val) -> boost::uuids::uuid { + auto uuid_from_string = [](char const * val) -> boost::uuids::uuid { return boost::lexical_cast(val); }; @@ -168,85 +186,87 @@ SCENARIO("Opening older files") { std::string pore_type; std::string run_info_id; }; + std::vector test_read_data{ - {{uuid_from_string("0000173c-bf67-44e7-9a9c-1ad0bc728e74")}, - 1093, - 21.0f, - 0.1755f, - "unknown", - "not_set", - "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, - {{uuid_from_string("002fde30-9e23-4125-9eae-d112c18a81a7")}, - 75, - 4.0f, - 0.1755f, - "unknown", - "not_set", - "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, - {{uuid_from_string("006d1319-2877-4b34-85df-34de7250a47b")}, - 1053, - 6.0f, - 0.1755f, - "unknown", - "not_set", - "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, - {{uuid_from_string("00728efb-2120-4224-87d8-580fbb0bd4b2")}, - 657, - 2.0f, - 0.1755f, - "unknown", - "not_set", - "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, - {{uuid_from_string("007cc97e-6de2-4ff6-a0fd-1c1eca816425")}, - 1625, - 23.0f, - 0.1755f, - "unknown", - "not_set", - "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, - {{uuid_from_string("008468c3-e477-46c4-a6e2-7d021a4ebf0b")}, - 411, - 4.0f, - 0.1755f, - "unknown", - "not_set", - "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, - {{uuid_from_string("008ed3dc-86c2-452f-b107-6877a473d177")}, - 513, - 5.0f, - 0.1755f, - "unknown", - "not_set", - "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, - {{uuid_from_string("00919556-e519-4960-8aa5-c2dfa020980c")}, - 56, - 2.0f, - 0.1755f, - "unknown", - "not_set", - "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, - {{uuid_from_string("00925f34-6baf-47fc-b40c-22591e27fb5c")}, - 930, - 37.0f, - 0.1755f, - "unknown", - "not_set", - "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, - {{uuid_from_string("009dc9bd-c5f4-487b-ba4c-b9ce7e3a711e")}, - 195, - 14.0f, - 0.1755f, - "unknown", - "not_set", - "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, + {{uuid_from_string("0000173c-bf67-44e7-9a9c-1ad0bc728e74")}, + 1093, + 21.0f, + 0.1755f, + "unknown", + "not_set", + "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, + {{uuid_from_string("002fde30-9e23-4125-9eae-d112c18a81a7")}, + 75, + 4.0f, + 0.1755f, + "unknown", + "not_set", + "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, + {{uuid_from_string("006d1319-2877-4b34-85df-34de7250a47b")}, + 1053, + 6.0f, + 0.1755f, + "unknown", + "not_set", + "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, + {{uuid_from_string("00728efb-2120-4224-87d8-580fbb0bd4b2")}, + 657, + 2.0f, + 0.1755f, + "unknown", + "not_set", + "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, + {{uuid_from_string("007cc97e-6de2-4ff6-a0fd-1c1eca816425")}, + 1625, + 23.0f, + 0.1755f, + "unknown", + "not_set", + "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, + {{uuid_from_string("008468c3-e477-46c4-a6e2-7d021a4ebf0b")}, + 411, + 4.0f, + 0.1755f, + "unknown", + "not_set", + "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, + {{uuid_from_string("008ed3dc-86c2-452f-b107-6877a473d177")}, + 513, + 5.0f, + 0.1755f, + "unknown", + "not_set", + "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, + {{uuid_from_string("00919556-e519-4960-8aa5-c2dfa020980c")}, + 56, + 2.0f, + 0.1755f, + "unknown", + "not_set", + "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, + {{uuid_from_string("00925f34-6baf-47fc-b40c-22591e27fb5c")}, + 930, + 37.0f, + 0.1755f, + "unknown", + "not_set", + "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, + {{uuid_from_string("009dc9bd-c5f4-487b-ba4c-b9ce7e3a711e")}, + 195, + 14.0f, + 0.1755f, + "unknown", + "not_set", + "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, }; auto repo_root = - ::arrow::internal::PlatformFilename::FromString(__FILE__)->Parent().Parent().Parent(); - auto path = GENERATE_COPY(*repo_root.Join("test_data/multi_fast5_zip_v0.pod5"), - *repo_root.Join("test_data/multi_fast5_zip_v1.pod5"), - *repo_root.Join("test_data/multi_fast5_zip_v2.pod5"), - *repo_root.Join("test_data/multi_fast5_zip_v3.pod5")); + ::arrow::internal::PlatformFilename::FromString(__FILE__)->Parent().Parent().Parent(); + auto path = GENERATE_COPY( + *repo_root.Join("test_data/multi_fast5_zip_v0.pod5"), + *repo_root.Join("test_data/multi_fast5_zip_v1.pod5"), + *repo_root.Join("test_data/multi_fast5_zip_v2.pod5"), + *repo_root.Join("test_data/multi_fast5_zip_v3.pod5")); auto reader = pod5::open_file_reader(path.ToString(), {}); CHECK_ARROW_STATUS_OK(reader); @@ -288,35 +308,36 @@ SCENARIO("Opening older files") { CHECK((*run_info)->adc_max == 4095); CHECK((*run_info)->protocol_run_id == "df049455-3552-438c-8176-d4a5b1dd9fc5"); CHECK((*run_info)->software == "python-pod5-converter"); - CHECK((*run_info)->tracking_id == - pod5::RunInfoData::MapType{ - {"asic_id", "131070"}, - {"asic_id_eeprom", "0"}, - {"asic_temp", "35.043102"}, - {"asic_version", "IA02C"}, - {"auto_update", "0"}, - {"auto_update_source", "https://mirror.oxfordnanoportal.com/software/MinKNOW/"}, - {"bream_is_standard", "0"}, - {"device_id", "MS00000"}, - {"device_type", "minion"}, - {"distribution_status", "modified"}, - {"distribution_version", "unknown"}, - {"exp_script_name", "c449127e3461a521e0865fe6a88716f6f6b0b30c"}, - {"exp_script_purpose", "sequencing_run"}, - {"exp_start_time", "2019-05-13T11:11:43Z"}, - {"flow_cell_id", ""}, - {"guppy_version", "3.0.3+7e7b7d0"}, - {"heatsink_temp", "35.000000"}, - {"hostname", "happy_fish"}, - {"installation_type", "prod"}, - {"local_firmware_file", "1"}, - {"operating_system", "ubuntu 16.04"}, - {"protocol_group_id", "TEST_EXPERIMENT"}, - {"protocol_run_id", "df049455-3552-438c-8176-d4a5b1dd9fc5"}, - {"protocols_version", "4.0.6"}, - {"run_id", "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, - {"sample_id", "TEST_SAMPLE"}, - {"usb_config", "MinION_fx3_1.1.1_ONT#MinION_fpga_1.1.0#ctrl#Auto"}, - {"version", "3.4.0-rc3"}, - }); + CHECK( + (*run_info)->tracking_id + == pod5::RunInfoData::MapType{ + {"asic_id", "131070"}, + {"asic_id_eeprom", "0"}, + {"asic_temp", "35.043102"}, + {"asic_version", "IA02C"}, + {"auto_update", "0"}, + {"auto_update_source", "https://mirror.oxfordnanoportal.com/software/MinKNOW/"}, + {"bream_is_standard", "0"}, + {"device_id", "MS00000"}, + {"device_type", "minion"}, + {"distribution_status", "modified"}, + {"distribution_version", "unknown"}, + {"exp_script_name", "c449127e3461a521e0865fe6a88716f6f6b0b30c"}, + {"exp_script_purpose", "sequencing_run"}, + {"exp_start_time", "2019-05-13T11:11:43Z"}, + {"flow_cell_id", ""}, + {"guppy_version", "3.0.3+7e7b7d0"}, + {"heatsink_temp", "35.000000"}, + {"hostname", "happy_fish"}, + {"installation_type", "prod"}, + {"local_firmware_file", "1"}, + {"operating_system", "ubuntu 16.04"}, + {"protocol_group_id", "TEST_EXPERIMENT"}, + {"protocol_run_id", "df049455-3552-438c-8176-d4a5b1dd9fc5"}, + {"protocols_version", "4.0.6"}, + {"run_id", "a08e850aaa44c8b56765eee10b386fc3e516a62b"}, + {"sample_id", "TEST_SAMPLE"}, + {"usb_config", "MinION_fx3_1.1.1_ONT#MinION_fpga_1.1.0#ctrl#Auto"}, + {"version", "3.4.0-rc3"}, + }); } diff --git a/c++/test/read_table_tests.cpp b/c++/test/read_table_tests.cpp index 1fba7de..8185de5 100644 --- a/c++/test/read_table_tests.cpp +++ b/c++/test/read_table_tests.cpp @@ -15,8 +15,10 @@ #include #include -bool operator==(std::shared_ptr const& array, - std::vector const& vec) { +bool operator==( + std::shared_ptr const & array, + std::vector const & vec) +{ if (array->length() != vec.size()) { return false; } @@ -29,7 +31,8 @@ bool operator==(std::shared_ptr const& array, return true; } -SCENARIO("Read table Tests") { +SCENARIO("Read table Tests") +{ using namespace pod5; (void)pod5::register_extension_types(); @@ -47,32 +50,33 @@ SCENARIO("Read table Tests") { std::copy(uuid_source.begin(), uuid_source.end(), read_id.begin()); return std::make_tuple( - pod5::ReadData{ - read_id, - std::uint32_t(index * 2), - std::uint64_t(index * 10), - std::uint16_t(index + 1), - std::uint8_t(index + 2), - 0, - index * 0.1f, - index * 0.2f, - index * 100.0f, - 0, - true, - 0, - std::uint64_t(index * 150), - index * 0.4f, - index * 0.3f, - index * 0.6f, - index * 0.5f, - std::uint32_t(index + 10), - index * 50.0f, - - }, - std::vector{index + 2, index + 3}); + pod5::ReadData{ + read_id, + std::uint32_t(index * 2), + std::uint64_t(index * 10), + std::uint16_t(index + 1), + std::uint8_t(index + 2), + 0, + index * 0.1f, + index * 0.2f, + index * 100.0f, + 0, + true, + 0, + std::uint64_t(index * 150), + index * 0.4f, + index * 0.3f, + index * 0.6f, + index * 0.5f, + std::uint32_t(index + 10), + index * 50.0f, + + }, + std::vector{index + 2, index + 3}); }; - GIVEN("A read table writer") { + GIVEN("A read table writer") + { auto filename = "./foo.pod5"; auto pool = arrow::system_memory_pool(); @@ -83,7 +87,7 @@ SCENARIO("Read table Tests") { { auto schema_metadata = make_schema_key_value_metadata( - {file_identifier, "test_software", *parse_version_number(Pod5Version)}); + {file_identifier, "test_software", *parse_version_number(Pod5Version)}); REQUIRE_ARROW_STATUS_OK(schema_metadata); REQUIRE_ARROW_STATUS_OK(file_out); @@ -94,9 +98,14 @@ SCENARIO("Read table Tests") { auto run_info_writer = pod5::make_run_info_writer(pool); REQUIRE_ARROW_STATUS_OK(run_info_writer); - auto writer = pod5::make_read_table_writer(*file_out, *schema_metadata, read_count, - *pore_writer, *end_reason_writer, - *run_info_writer, pool); + auto writer = pod5::make_read_table_writer( + *file_out, + *schema_metadata, + read_count, + *pore_writer, + *end_reason_writer, + *run_info_writer, + pool); REQUIRE_ARROW_STATUS_OK(writer); auto const pore_1 = (*pore_writer)->add("Well Type"); @@ -160,11 +169,11 @@ SCENARIO("Read table Tests") { CHECK(columns->run_info->length() == read_count); auto pore_indices = - std::static_pointer_cast(columns->pore_type->indices()); + std::static_pointer_cast(columns->pore_type->indices()); auto end_reason_indices = - std::static_pointer_cast(columns->end_reason->indices()); + std::static_pointer_cast(columns->end_reason->indices()); auto run_info_indices = - std::static_pointer_cast(columns->run_info->indices()); + std::static_pointer_cast(columns->run_info->indices()); for (auto j = 0; j < read_count; ++j) { auto idx = j + i * read_count; @@ -175,9 +184,10 @@ SCENARIO("Read table Tests") { CHECK(columns->read_id->Value(j) == read_data.read_id); auto signal_data = std::static_pointer_cast( - columns->signal->value_slice(j)); - CHECK(gsl::make_span(signal_data->raw_values(), signal_data->length()) == - gsl::make_span(expected_signal)); + columns->signal->value_slice(j)); + CHECK( + gsl::make_span(signal_data->raw_values(), signal_data->length()) + == gsl::make_span(expected_signal)); CHECK(columns->read_number->Value(j) == read_data.read_number); CHECK(columns->start_sample->Value(j) == read_data.start_sample); diff --git a/c++/test/read_table_writer_utils_tests.cpp b/c++/test/read_table_writer_utils_tests.cpp index 82abe89..00089a8 100644 --- a/c++/test/read_table_writer_utils_tests.cpp +++ b/c++/test/read_table_writer_utils_tests.cpp @@ -10,7 +10,8 @@ #include template -std::shared_ptr get_field(arrow::StructArray& struct_array, char const* name) { +std::shared_ptr get_field(arrow::StructArray & struct_array, char const * name) +{ CAPTURE(name); auto field = struct_array.GetFieldByName(name); REQUIRE(field); @@ -19,70 +20,85 @@ std::shared_ptr get_field(arrow::StructArray& struct_array, char const* name) return typed_field; } -void check_field(std::size_t index, - arrow::StructArray& struct_array, - char const* name, - std::uint16_t data) { +void check_field( + std::size_t index, + arrow::StructArray & struct_array, + char const * name, + std::uint16_t data) +{ INFO("name " << name << ", index " << index); auto field = get_field(struct_array, name); CHECK((*field)[index] == data); } -void check_field(std::size_t index, - arrow::StructArray& struct_array, - char const* name, - std::int16_t data) { +void check_field( + std::size_t index, + arrow::StructArray & struct_array, + char const * name, + std::int16_t data) +{ INFO("name " << name << ", index " << index); auto field = get_field(struct_array, name); CHECK((*field)[index] == data); } -void check_field(std::size_t index, - arrow::StructArray& struct_array, - char const* name, - std::uint8_t data) { +void check_field( + std::size_t index, + arrow::StructArray & struct_array, + char const * name, + std::uint8_t data) +{ INFO("name " << name << ", index " << index); auto field = get_field(struct_array, name); CHECK((*field)[index] == data); } -void check_field(std::size_t index, - arrow::StructArray& struct_array, - char const* name, - float data) { +void check_field( + std::size_t index, + arrow::StructArray & struct_array, + char const * name, + float data) +{ INFO("name " << name << ", index " << index); auto field = get_field(struct_array, name); CHECK((*field)[index] == data); } -void check_field(std::size_t index, arrow::StructArray& struct_array, char const* name, bool data) { +void check_field(std::size_t index, arrow::StructArray & struct_array, char const * name, bool data) +{ INFO("name " << name << ", index " << index); auto field = get_field(struct_array, name); CHECK((*field)[index] == data); } -void check_field(std::size_t index, - arrow::StructArray& struct_array, - char const* name, - std::string const& data) { +void check_field( + std::size_t index, + arrow::StructArray & struct_array, + char const * name, + std::string const & data) +{ INFO("name " << name << ", index " << index); auto field = get_field(struct_array, name); CHECK((*field)[index] == data); } -void check_timestamp_field(std::size_t index, - arrow::StructArray& struct_array, - char const* name, - std::int64_t milliseconds_since_epoch) { +void check_timestamp_field( + std::size_t index, + arrow::StructArray & struct_array, + char const * name, + std::int64_t milliseconds_since_epoch) +{ INFO("name " << name << ", index " << index); auto field = get_field(struct_array, name); CHECK((*field)[index] == milliseconds_since_epoch); } -void check_field(std::size_t index, - arrow::StructArray& struct_array, - char const* name, - pod5::RunInfoData::MapType const& data) { +void check_field( + std::size_t index, + arrow::StructArray & struct_array, + char const * name, + pod5::RunInfoData::MapType const & data) +{ auto field = get_field(struct_array, name); auto offsets = std::dynamic_pointer_cast(field->offsets()); @@ -102,7 +118,8 @@ void check_field(std::size_t index, CHECK(extracted_data == data); } -TEST_CASE("Run Info Writer Tests") { +TEST_CASE("Run Info Writer Tests") +{ auto pool = arrow::system_memory_pool(); auto run_info_writer = pod5::make_run_info_writer(pool); REQUIRE_ARROW_STATUS_OK(run_info_writer); @@ -114,7 +131,8 @@ TEST_CASE("Run Info Writer Tests") { // Important to always call this so we test calling it twice auto const value_array = (*run_info_writer)->get_value_array(); - WHEN("Checking the first row") { + WHEN("Checking the first row") + { REQUIRE_ARROW_STATUS_OK(value_array); auto string_value_array = std::dynamic_pointer_cast(*value_array); @@ -128,7 +146,8 @@ TEST_CASE("Run Info Writer Tests") { CHECK(*index == 1); CHECK((*run_info_writer)->item_count() == 2); - WHEN("Checking the rows after a second append") { + WHEN("Checking the rows after a second append") + { auto value_array = (*run_info_writer)->get_value_array(); REQUIRE_ARROW_STATUS_OK(value_array); diff --git a/c++/test/run_info_table_tests.cpp b/c++/test/run_info_table_tests.cpp index fd66731..fb52aad 100644 --- a/c++/test/run_info_table_tests.cpp +++ b/c++/test/run_info_table_tests.cpp @@ -15,7 +15,8 @@ #include #include -SCENARIO("Run Info table Tests") { +SCENARIO("Run Info table Tests") +{ using namespace pod5; (void)pod5::register_extension_types(); @@ -25,7 +26,8 @@ SCENARIO("Run Info table Tests") { auto file_identifier = uuid_gen(); - GIVEN("A read table writer") { + GIVEN("A read table writer") + { auto filename = "./foo.pod5"; auto pool = arrow::system_memory_pool(); @@ -36,14 +38,14 @@ SCENARIO("Run Info table Tests") { { auto schema_metadata = make_schema_key_value_metadata( - {file_identifier, "test_software", *parse_version_number(Pod5Version)}); + {file_identifier, "test_software", *parse_version_number(Pod5Version)}); REQUIRE_ARROW_STATUS_OK(schema_metadata); REQUIRE_ARROW_STATUS_OK(file_out); std::size_t run_info_per_batch = 2; - auto writer = pod5::make_run_info_table_writer(*file_out, *schema_metadata, - run_info_per_batch, pool); + auto writer = pod5::make_run_info_table_writer( + *file_out, *schema_metadata, run_info_per_batch, pool); REQUIRE_ARROW_STATUS_OK(writer); REQUIRE_ARROW_STATUS_OK(writer->add_run_info(run_info_data_0)); @@ -70,28 +72,32 @@ SCENARIO("Run Info table Tests") { auto columns = record_batch->columns(); REQUIRE_ARROW_STATUS_OK(columns); - auto check_run_info = [](auto& columns, std::size_t index, - pod5::RunInfoData const& run_info_data) { + auto check_run_info = [](auto & columns, + std::size_t index, + pod5::RunInfoData const & run_info_data) { CHECK(columns.acquisition_id->Value(index) == run_info_data.acquisition_id); - CHECK(columns.acquisition_start_time->Value(index) == - run_info_data.acquisition_start_time); + CHECK( + columns.acquisition_start_time->Value(index) + == run_info_data.acquisition_start_time); CHECK(columns.adc_max->Value(index) == run_info_data.adc_max); CHECK(columns.adc_min->Value(index) == run_info_data.adc_min); CHECK(columns.experiment_name->Value(index) == run_info_data.experiment_name); CHECK(columns.flow_cell_id->Value(index) == run_info_data.flow_cell_id); - CHECK(columns.flow_cell_product_code->Value(index) == - run_info_data.flow_cell_product_code); + CHECK( + columns.flow_cell_product_code->Value(index) + == run_info_data.flow_cell_product_code); CHECK(columns.protocol_name->Value(index) == run_info_data.protocol_name); CHECK(columns.protocol_run_id->Value(index) == run_info_data.protocol_run_id); - CHECK(columns.protocol_start_time->Value(index) == - run_info_data.protocol_start_time); + CHECK( + columns.protocol_start_time->Value(index) == run_info_data.protocol_start_time); CHECK(columns.sample_id->Value(index) == run_info_data.sample_id); CHECK(columns.sample_rate->Value(index) == run_info_data.sample_rate); CHECK(columns.sequencing_kit->Value(index) == run_info_data.sequencing_kit); CHECK(columns.sequencer_position->Value(index) == run_info_data.sequencer_position); - CHECK(columns.sequencer_position_type->Value(index) == - run_info_data.sequencer_position_type); + CHECK( + columns.sequencer_position_type->Value(index) + == run_info_data.sequencer_position_type); CHECK(columns.software->Value(index) == run_info_data.software); CHECK(columns.system_name->Value(index) == run_info_data.system_name); CHECK(columns.system_type->Value(index) == run_info_data.system_type); diff --git a/c++/test/schema_tests.cpp b/c++/test/schema_tests.cpp index cafdc2e..d9116e1 100644 --- a/c++/test/schema_tests.cpp +++ b/c++/test/schema_tests.cpp @@ -3,7 +3,8 @@ #include -SCENARIO("Version Tests") { +SCENARIO("Version Tests") +{ using namespace pod5; CHECK(Version(1, 2, 3) < Version(3, 2, 1)); diff --git a/c++/test/signal_compression_tests.cpp b/c++/test/signal_compression_tests.cpp index b7f8dc3..05b4a46 100644 --- a/c++/test/signal_compression_tests.cpp +++ b/c++/test/signal_compression_tests.cpp @@ -10,7 +10,8 @@ #include -SCENARIO("Signal compression Tests") { +SCENARIO("Signal compression Tests") +{ auto pool = arrow::system_memory_pool(); std::vector signal(100'00); @@ -23,7 +24,7 @@ SCENARIO("Signal compression Tests") { auto decompressed = pod5::decompress_signal(compressed_span, signal.size(), pool); REQUIRE_ARROW_STATUS_OK(decompressed); auto decompressed_span = gsl::make_span((*decompressed)->data(), (*decompressed)->size()) - .as_span(); + .as_span(); CHECK(gsl::make_span(signal) == decompressed_span); } diff --git a/c++/test/signal_table_tests.cpp b/c++/test/signal_table_tests.cpp index 079777e..4813212 100644 --- a/c++/test/signal_table_tests.cpp +++ b/c++/test/signal_table_tests.cpp @@ -17,7 +17,8 @@ #include -SCENARIO("Signal table Tests") { +SCENARIO("Signal table Tests") +{ using namespace pod5; (void)pod5::register_extension_types(); @@ -33,7 +34,8 @@ SCENARIO("Signal table Tests") { std::iota(signal_1.begin(), signal_1.end(), 0); std::vector signal_2(10'000, 1); - GIVEN("A signal table writer") { + GIVEN("A signal table writer") + { auto filename = "./foo.pod5"; auto pool = arrow::system_memory_pool(); @@ -43,22 +45,24 @@ SCENARIO("Signal table Tests") { { auto schema_metadata = make_schema_key_value_metadata( - {file_identifier, "test_software", *parse_version_number(Pod5Version)}); + {file_identifier, "test_software", *parse_version_number(Pod5Version)}); REQUIRE_ARROW_STATUS_OK(schema_metadata); REQUIRE_ARROW_STATUS_OK(file_out); - auto writer = pod5::make_signal_table_writer(*file_out, *schema_metadata, 100, - signal_type, pool); + auto writer = + pod5::make_signal_table_writer(*file_out, *schema_metadata, 100, signal_type, pool); REQUIRE_ARROW_STATUS_OK(writer); - WHEN("Writing a read") { + WHEN("Writing a read") + { auto row_1 = writer->add_signal(read_id_1, gsl::make_span(signal_1)); auto row_2 = writer->add_signal(read_id_2, gsl::make_span(signal_2)); REQUIRE_ARROW_STATUS_OK(writer->close()); - THEN("Read row ids are correct") { + THEN("Read row ids are correct") + { REQUIRE_ARROW_STATUS_OK(row_1); REQUIRE_ARROW_STATUS_OK(row_2); CHECK(*row_1 == 0); @@ -95,17 +99,17 @@ SCENARIO("Signal table Tests") { CHECK(signal->length() == 2); auto compare_compressed_signal = - [&](gsl::span compressed_actual, - std::vector const& expected) { - auto decompressed = pod5::decompress_signal(compressed_actual, - expected.size(), pool); - REQUIRE_ARROW_STATUS_OK(decompressed); - - auto actual = - gsl::make_span((*decompressed)->data(), (*decompressed)->size()) - .as_span(); - CHECK(actual == gsl::make_span(expected)); - }; + [&](gsl::span compressed_actual, + std::vector const & expected) { + auto decompressed = + pod5::decompress_signal(compressed_actual, expected.size(), pool); + REQUIRE_ARROW_STATUS_OK(decompressed); + + auto actual = + gsl::make_span((*decompressed)->data(), (*decompressed)->size()) + .as_span(); + CHECK(actual == gsl::make_span(expected)); + }; auto signal_typed = std::static_pointer_cast(signal); compare_compressed_signal(signal_typed->Value(0), signal_1); @@ -115,16 +119,16 @@ SCENARIO("Signal table Tests") { CHECK(signal->length() == 2); auto signal_1_read = - std::static_pointer_cast(signal->value_slice(0)); + std::static_pointer_cast(signal->value_slice(0)); std::vector stored_values_1( - signal_1_read->raw_values(), - signal_1_read->raw_values() + signal_1_read->length()); + signal_1_read->raw_values(), + signal_1_read->raw_values() + signal_1_read->length()); CHECK(stored_values_1 == signal_1); auto signal_2_read = - std::static_pointer_cast(signal->value_slice(1)); + std::static_pointer_cast(signal->value_slice(1)); std::vector stored_values_2( - signal_2_read->raw_values(), - signal_2_read->raw_values() + signal_2_read->length()); + signal_2_read->raw_values(), + signal_2_read->raw_values() + signal_2_read->length()); CHECK(stored_values_2 == signal_2); } else { FAIL("Unknown signal type"); diff --git a/c++/test/svb16_scalar_tests.cpp b/c++/test/svb16_scalar_tests.cpp index 369aee9..d9fe1e2 100644 --- a/c++/test/svb16_scalar_tests.cpp +++ b/c++/test/svb16_scalar_tests.cpp @@ -12,56 +12,67 @@ using Catch::Matchers::Equals; template -void test_scalar_encode_scalar_decode() { +void test_scalar_encode_scalar_decode() +{ static constexpr uint32_t DATA_COUNT = 1024; std::minstd_rand rng; std::vector data(DATA_COUNT); - std::uniform_int_distribution dist{std::numeric_limits::min(), - std::numeric_limits::max()}; + std::uniform_int_distribution dist{ + std::numeric_limits::min(), std::numeric_limits::max()}; std::generate(data.begin(), data.end(), [&] { return dist(rng); }); std::vector encoded(svb16_max_encoded_length(data.size())); - auto const encoded_count = svb16::encode_scalar( - data.data(), encoded.data(), - encoded.data() + svb16_key_length(data.size()), DATA_COUNT) - - encoded.data(); + auto const encoded_count = + svb16::encode_scalar( + data.data(), encoded.data(), encoded.data() + svb16_key_length(data.size()), DATA_COUNT) + - encoded.data(); CHECK(encoded_count <= svb16_max_encoded_length(data.size())); std::vector decoded(DATA_COUNT); auto const consumed = svb16::decode_scalar( - decoded.data(), encoded.data(), - encoded.data() + svb16_key_length(data.size()), DATA_COUNT) - - encoded.data(); + decoded.data(), + encoded.data(), + encoded.data() + svb16_key_length(data.size()), + DATA_COUNT) + - encoded.data(); CHECK(consumed == encoded_count); CHECK_THAT(decoded, Equals(data)); } -TEST_CASE("Scalar decode is inverse of scalar encode", "[scalar]") { - SECTION("Unsigned, no delta, no zig-zag") { +TEST_CASE("Scalar decode is inverse of scalar encode", "[scalar]") +{ + SECTION("Unsigned, no delta, no zig-zag") + { test_scalar_encode_scalar_decode(); } - SECTION("Signed, no delta, no zig-zag") { + SECTION("Signed, no delta, no zig-zag") + { test_scalar_encode_scalar_decode(); } - SECTION("Unsigned, delta, no zig-zag") { + SECTION("Unsigned, delta, no zig-zag") + { test_scalar_encode_scalar_decode(); } - SECTION("Signed, delta, no zig-zag") { + SECTION("Signed, delta, no zig-zag") + { test_scalar_encode_scalar_decode(); } - SECTION("Unsigned, delta, zig-zag") { + SECTION("Unsigned, delta, zig-zag") + { test_scalar_encode_scalar_decode(); } SECTION("Signed, delta, zig-zag") { test_scalar_encode_scalar_decode(); } - SECTION("Unsigned, no delta, zig-zag") { + SECTION("Unsigned, no delta, zig-zag") + { // this scenario doesn't really make sense, but it's possible, so let's test it test_scalar_encode_scalar_decode(); } - SECTION("Signed, no delta, zig-zag") { + SECTION("Signed, no delta, zig-zag") + { test_scalar_encode_scalar_decode(); } } diff --git a/c++/test/svb16_x64_tests.cpp b/c++/test/svb16_x64_tests.cpp index 69f8c5f..27d239e 100644 --- a/c++/test/svb16_x64_tests.cpp +++ b/c++/test/svb16_x64_tests.cpp @@ -15,58 +15,67 @@ using Catch::Matchers::Equals; template -void test_sse_encode_scalar_decode() { +void test_sse_encode_scalar_decode() +{ uint32_t const DATA_COUNT = GENERATE( - 1000, - 20000); // Deliberately not aligned to 64 so we test the scalar tidy up code at the end. + 1000, + 20000); // Deliberately not aligned to 64 so we test the scalar tidy up code at the end. std::minstd_rand rng; std::vector data(DATA_COUNT); - std::uniform_int_distribution dist{std::numeric_limits::min(), - std::numeric_limits::max()}; + std::uniform_int_distribution dist{ + std::numeric_limits::min(), std::numeric_limits::max()}; std::generate(data.begin(), data.end(), [&] { return dist(rng); }); std::vector encoded(svb16_max_encoded_length(data.size())); - auto const encoded_count = svb16::encode_sse( - data.data(), encoded.data(), - encoded.data() + svb16_key_length(data.size()), DATA_COUNT) - - encoded.data(); + auto const encoded_count = + svb16::encode_sse( + data.data(), encoded.data(), encoded.data() + svb16_key_length(data.size()), DATA_COUNT) + - encoded.data(); CHECK(encoded_count <= svb16_max_encoded_length(data.size())); std::vector encoded_scalar(svb16_max_encoded_length(data.size())); - auto const scalar_encoded_count = - svb16::encode_scalar( - data.data(), encoded_scalar.data(), - encoded_scalar.data() + svb16_key_length(data.size()), DATA_COUNT) - - encoded_scalar.data(); + auto const scalar_encoded_count = svb16::encode_scalar( + data.data(), + encoded_scalar.data(), + encoded_scalar.data() + svb16_key_length(data.size()), + DATA_COUNT) + - encoded_scalar.data(); CHECK(scalar_encoded_count == encoded_count); CHECK(encoded == encoded_scalar); std::vector decoded(DATA_COUNT); auto const consumed = svb16::decode_sse( - decoded.data(), encoded.data(), - encoded.data() + svb16_key_length(data.size()), DATA_COUNT) - - encoded.data(); + decoded.data(), + encoded.data(), + encoded.data() + svb16_key_length(data.size()), + DATA_COUNT) + - encoded.data(); CHECK(consumed == encoded_count); CHECK_THAT(decoded, Equals(data)); } -TEST_CASE("SSE decode is inverse of scalar encode", "[scalar]") { - SECTION("Unsigned, no delta, no zig-zag") { +TEST_CASE("SSE decode is inverse of scalar encode", "[scalar]") +{ + SECTION("Unsigned, no delta, no zig-zag") + { test_sse_encode_scalar_decode(); } - SECTION("Signed, no delta, no zig-zag") { + SECTION("Signed, no delta, no zig-zag") + { test_sse_encode_scalar_decode(); } - SECTION("Unsigned, delta, no zig-zag") { + SECTION("Unsigned, delta, no zig-zag") + { test_sse_encode_scalar_decode(); } SECTION("Signed, delta, no zig-zag") { test_sse_encode_scalar_decode(); } SECTION("Unsigned, delta, zig-zag") { test_sse_encode_scalar_decode(); } SECTION("Signed, delta, zig-zag") { test_sse_encode_scalar_decode(); } - SECTION("Unsigned, no delta, zig-zag") { + SECTION("Unsigned, no delta, zig-zag") + { // this scenario doesn't really make sense, but it's possible, so let's test it test_sse_encode_scalar_decode(); } diff --git a/c++/test/test_utils.h b/c++/test/test_utils.h index 0bf79d8..2db2d30 100644 --- a/c++/test/test_utils.h +++ b/c++/test/test_utils.h @@ -8,7 +8,7 @@ class IsStatusOk : public Catch::MatcherBase { public: IsStatusOk() = default; - bool match(arrow::Status const& status) const override { return status.ok() == CheckOk; } + bool match(arrow::Status const & status) const override { return status.ok() == CheckOk; } virtual std::string describe() const override { return "== arrow::Status::OK()"; } }; @@ -18,42 +18,44 @@ class IsResultOk : public Catch::MatcherBase> { public: IsResultOk() = default; - bool match(arrow::Result const& status) const override { return status.ok() == CheckOk; } + bool match(arrow::Result const & status) const override { return status.ok() == CheckOk; } virtual std::string describe() const override { return "== arrow::Status::OK()"; } }; template -inline IsResultOk _is_arrow_ok(arrow::Result const&) { +inline IsResultOk _is_arrow_ok(arrow::Result const &) +{ return IsResultOk(); } -inline IsStatusOk _is_arrow_ok(arrow::Status const&) { return IsStatusOk(); } +inline IsStatusOk _is_arrow_ok(arrow::Status const &) { return IsStatusOk(); } template -inline IsResultOk _is_arrow_not_ok(arrow::Result const&) { +inline IsResultOk _is_arrow_not_ok(arrow::Result const &) +{ return IsResultOk(); } -inline IsStatusOk _is_arrow_not_ok(arrow::Status const&) { return IsStatusOk(); } +inline IsStatusOk _is_arrow_not_ok(arrow::Status const &) { return IsStatusOk(); } #define CHECK_ARROW_STATUS_OK(statement) \ { \ - auto const& _res = (statement); \ + auto const & _res = (statement); \ CHECK_THAT(_res, _is_arrow_ok(_res)); \ } #define REQUIRE_ARROW_STATUS_OK(statement) \ { \ - auto const& _res = (statement); \ + auto const & _res = (statement); \ REQUIRE_THAT(_res, _is_arrow_ok(_res)); \ } #define CHECK_ARROW_STATUS_NOT_OK(statement) \ { \ - auto const& _res = (statement); \ + auto const & _res = (statement); \ CHECK_THAT(_res, _is_arrow_not_ok(_res)); \ } #define REQUIRE_ARROW_STATUS_NOT_OK(statement) \ { \ - auto const& _res = (statement); \ + auto const & _res = (statement); \ REQUIRE_THAT(_res, _is_arrow_not_ok(_res)); \ } diff --git a/c++/test/utils.h b/c++/test/utils.h index 82df7c3..4c897f9 100644 --- a/c++/test/utils.h +++ b/c++/test/utils.h @@ -9,28 +9,44 @@ namespace Catch { template struct StringMaker> { - static std::string convert(arrow::Result const& value) { return value.status().ToString(); } + static std::string convert(arrow::Result const & value) { return value.status().ToString(); } }; } // namespace Catch -inline pod5::RunInfoData get_test_run_info_data(std::string suffix = "", - std::int16_t adc_center_offset = 0, - std::int16_t sample_rate = 4000) { +inline pod5::RunInfoData get_test_run_info_data( + std::string suffix = "", + std::int16_t adc_center_offset = 0, + std::int16_t sample_rate = 4000) +{ return pod5::RunInfoData( - "acquisition_id" + suffix, 1005, 4095 + adc_center_offset, -4096 + adc_center_offset, - {{"context" + suffix, "tags" + suffix}, - {"other" + suffix, "tagz" + suffix}, - {"third" + suffix, "thing" + suffix}}, - "experiment_name" + suffix, "flow_cell_id" + suffix, "flow_cell_product_code" + suffix, - "protocol_name" + suffix, "protocol_run_id" + suffix, 200005, "sample_id" + suffix, - sample_rate, "sequencing_kit" + suffix, "sequencer_position" + suffix, - "sequencer_position_type" + suffix, "software" + suffix, "system_name" + suffix, - "system_type" + suffix, {{"tracking" + suffix, "id" + suffix}}); + "acquisition_id" + suffix, + 1005, + 4095 + adc_center_offset, + -4096 + adc_center_offset, + {{"context" + suffix, "tags" + suffix}, + {"other" + suffix, "tagz" + suffix}, + {"third" + suffix, "thing" + suffix}}, + "experiment_name" + suffix, + "flow_cell_id" + suffix, + "flow_cell_product_code" + suffix, + "protocol_name" + suffix, + "protocol_run_id" + suffix, + 200005, + "sample_id" + suffix, + sample_rate, + "sequencing_kit" + suffix, + "sequencer_position" + suffix, + "sequencer_position_type" + suffix, + "software" + suffix, + "system_name" + suffix, + "system_type" + suffix, + {{"tracking" + suffix, "id" + suffix}}); } -inline arrow::Status remove_file_if_exists(std::string const& file) { - ARROW_ASSIGN_OR_RAISE(auto arrow_reads_path, - ::arrow::internal::PlatformFilename::FromString(file)); +inline arrow::Status remove_file_if_exists(std::string const & file) +{ + ARROW_ASSIGN_OR_RAISE( + auto arrow_reads_path, ::arrow::internal::PlatformFilename::FromString(file)); ARROW_ASSIGN_OR_RAISE(bool file_exists, arrow::internal::FileExists(arrow_reads_path)); if (file_exists) { ARROW_RETURN_NOT_OK(arrow::internal::DeleteFile(arrow_reads_path)); diff --git a/test_package/test_package.cpp b/test_package/test_package.cpp index e9fabe0..35bd5d4 100644 --- a/test_package/test_package.cpp +++ b/test_package/test_package.cpp @@ -1,6 +1,7 @@ #include "pod5_format/c_api.h" -int main() { +int main() +{ pod5_init(); pod5_terminate(); }