Merge pull request #183 from ecmwf/feature/GRIB_lightweight_clone

Feature/grib lightweight clone

src/eccodes.cc

@@ -175,6 +175,11 @@ grib_handle* codes_handle_clone(const grib_handle* h)
 {
     return grib_handle_clone(h);
 }
+grib_handle* codes_handle_clone_headers_only(const grib_handle* h)
+{
+    return grib_handle_clone_headers_only(h);
+}
+
 int codes_handle_delete(grib_handle* h)
 {
     return grib_handle_delete(h);

src/eccodes.h

@@ -495,6 +495,7 @@ codes_handle* codes_handle_new_from_samples(codes_context* c, const char* sample
  * @return            the new handle, NULL if the message is invalid or a problem is encountered
  */
 codes_handle* codes_handle_clone(const codes_handle* h);
+codes_handle* codes_handle_clone_headers_only(const codes_handle* h);
 
 /**
  *  Frees a handle, also frees the message if it is not a user message

src/grib_api.h

@@ -507,6 +507,7 @@ grib_handle* grib_handle_new_from_samples(grib_context* c, const char* sample_na
  * @return            the new handle, NULL if the message is invalid or a problem is encountered
  */
 grib_handle* grib_handle_clone(const grib_handle* h);
+grib_handle* grib_handle_clone_headers_only(const grib_handle* h);
 
 /**
  *  Frees a handle, also frees the message if it is not a user message

src/grib_handle.cc

@@ -326,6 +326,104 @@ grib_handle* grib_handle_clone(const grib_handle* h)
     return result;
 }
 
+static bool can_create_clone_headers_only(const grib_handle* h)
+{
+    // Only for GRIB, not BUFR etc
+    if (h->product_kind != PRODUCT_GRIB) return false;
+
+    // Spectral data does not have constant fields!
+    long isGridded = 0;
+    int err = grib_get_long(h, "isGridded", &isGridded);
+    if (err || !isGridded) return false;
+
+    return true;
+}
+
+grib_handle* grib_handle_clone_headers_only(const grib_handle* h)
+{
+    int err = 0;
+    grib_handle* result = NULL;
+    grib_context* c = h->context;
+
+    if (!can_create_clone_headers_only(h)) {
+        // Headers-only clone not possible. Do a normal clone
+        return grib_handle_clone(h);
+    }
+
+    char sample_name[1024];
+    long edition = 0;
+    grib_get_long(h, "edition", &edition);
+    snprintf(sample_name, sizeof(sample_name), "GRIB%ld", edition);
+    grib_handle* h_sample = grib_handle_new_from_samples(c, sample_name);
+    if (!h_sample) {
+        grib_context_log(c, GRIB_LOG_ERROR, "Failed to create headers_only clone using sample %s", sample_name);
+        return NULL;
+    }
+
+    // Must preserve the packingType
+    char input_packing_type[100];
+    size_t len = sizeof(input_packing_type);
+    err = grib_get_string(h, "packingType", input_packing_type, &len);
+    if (!err) {
+        grib_set_string(h_sample, "packingType", input_packing_type, &len);
+    }
+
+    // Copy all sections except Bitmap and Data from h to h_sample
+    const int sections_to_copy = GRIB_SECTION_PRODUCT | GRIB_SECTION_LOCAL | GRIB_SECTION_GRID;
+    result = grib_util_sections_copy((grib_handle*)h, h_sample, sections_to_copy, &err);
+    if (!result || err) {
+        grib_context_log(c, GRIB_LOG_ERROR, "Failed to create headers_only clone: Unable to copy sections");
+        grib_handle_delete(h_sample);
+        return NULL;
+    }
+
+    grib_handle_delete(h_sample);
+    return result;
+}
+
+// grib_handle* grib_handle_clone_lightweight(const grib_handle* h)
+// {
+//     int err = 0;
+//     size_t size1 = 0;
+//     const void* msg1 = NULL;
+//     long edition = 0;
+
+//     // Only for GRIB, not BUFR etc
+//     if (h->product_kind != PRODUCT_GRIB) {
+//         grib_context_log(h->context, GRIB_LOG_ERROR, "%s: Only supported for %s",
+//                          __func__, codes_get_product_name(PRODUCT_GRIB));
+//         return NULL;
+//     }
+
+//     err = grib_get_long(h, "edition", &edition);
+//     if (!err && edition == 1) {
+//         grib_context_log(h->context, GRIB_LOG_ERROR, "%s: Edition not supported", __func__);
+//         return NULL;
+//     }
+
+//     err = grib_get_message_headers(h, &msg1, &size1);
+//     if (err) return NULL;
+
+//     size1 += 4;
+//     grib_handle* result  = grib_handle_new_from_partial_message_copy(h->context, msg1, size1);
+//     result->buffer->data[ size1 - 4 ] = '7';
+//     result->buffer->data[ size1 - 3 ] = '7';
+//     result->buffer->data[ size1 - 2 ] = '7';
+//     result->buffer->data[ size1 - 1 ] = '7';
+//     result->buffer->ulength = size1;
+
+//     result->product_kind = h->product_kind;
+
+//     long off = 64; // This is only true for GRIB edition 2
+//     err = grib_encode_unsigned_long( result->buffer->data, (unsigned long)size1, &off, 64);
+//     if (err) {
+//         printf("err=%s\n", grib_get_error_message(err));
+//         return NULL;
+//     }
+
+//     return result;
+// }
+
 grib_handle* codes_handle_new_from_file(grib_context* c, FILE* f, ProductKind product, int* error)
 {
     if (product == PRODUCT_GRIB)

tests/CMakeLists.txt

@@ -17,6 +17,7 @@ list(APPEND test_c_bins
     grib_indexing
     grib_fieldset
     grib_multi_from_message
+    grib_clone_headers_only
     grib_read_index
     unit_tests
     bufr_keys_iter
@@ -171,6 +172,7 @@ if( HAVE_BUILD_TOOLS )
         grib_headers_only
         grib_unpack_subarray
         grib_count
+        grib_clone_headers_only
         bufr_templates
         bufr_dump_data
         bufr_dump_descriptors

tests/grib_clone_headers_only.cc

@@ -0,0 +1,90 @@
+/*
+ * (C) Copyright 2005- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
+ * virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
+ */
+#include <stdio.h>
+#include "eccodes.h"
+#undef NDEBUG
+#include <assert.h>
+
+static void usage(const char* app)
+{
+    fprintf(stderr, "Usage is: %s input_file ouput_file\n", app);
+}
+
+int main(int argc, char* argv[])
+{
+    FILE* in                    = NULL;
+    FILE* out                   = NULL;
+    codes_handle* source_handle = NULL;
+    const void* buffer          = NULL;
+    int err                     = 0;
+
+    long totalLength_src = 0, totalLength_dst = 0;
+    long edition = 0, isGridded = 0, bitmapPresent = 0;
+    long isConstant_src = 0, isConstant_dst = 0;
+    long dataSectionLength_src = 0, dataSectionLength_dst = 0;
+    size_t messageLength_src = 0, messageLength_dst = 0;
+
+    if (argc != 3) {
+        usage(argv[0]);
+        return 1;
+    }
+
+    in  = fopen(argv[1], "rb");
+    assert(in);
+    out = fopen(argv[2], "wb");
+    assert(out);
+
+    while ((source_handle = codes_handle_new_from_file(0, in, PRODUCT_GRIB, &err)) != NULL) {
+        codes_handle* clone_handle = codes_handle_clone_headers_only(source_handle);
+        assert(clone_handle);
+
+        codes_get_long(source_handle, "isConstant", &isConstant_src);
+        codes_get_long(source_handle, "isGridded", &isGridded);
+        if (isGridded && !isConstant_src) {
+
+            CODES_CHECK(codes_get_message(source_handle, &buffer, &messageLength_src), 0);
+            CODES_CHECK(codes_get_message(clone_handle, &buffer, &messageLength_dst), 0);
+            assert( messageLength_src > messageLength_dst );
+
+            CODES_CHECK(codes_get_long(source_handle, "totalLength", &totalLength_src), 0);
+            CODES_CHECK(codes_get_long(clone_handle, "totalLength", &totalLength_dst), 0);
+            assert(totalLength_src > totalLength_dst);
+
+            CODES_CHECK(codes_get_long(source_handle, "edition", &edition), 0);
+            if (edition == 1) {
+                CODES_CHECK(codes_get_long(source_handle, "section4Length", &dataSectionLength_src), 0);
+                CODES_CHECK(codes_get_long(clone_handle, "section4Length", &dataSectionLength_dst), 0);
+            } else if (edition == 2) {
+                CODES_CHECK(codes_get_long(source_handle, "section7Length", &dataSectionLength_src), 0);
+                CODES_CHECK(codes_get_long(clone_handle, "section7Length", &dataSectionLength_dst), 0);
+            }
+            assert( dataSectionLength_src > dataSectionLength_dst );
+
+            codes_get_long(clone_handle, "bitmapPresent", &bitmapPresent);
+            assert(bitmapPresent == 0);
+            codes_get_long(clone_handle, "isConstant", &isConstant_dst);
+            assert(isConstant_dst == 1);
+        }
+
+        /* write out the cloned buffer */
+        if (fwrite(buffer, 1, messageLength_dst, out) != messageLength_dst) {
+            perror(argv[1]);
+            return 1;
+        }
+        codes_handle_delete(clone_handle);
+        codes_handle_delete(source_handle);
+    }
+
+    fclose(out);
+    fclose(in);
+
+    printf("All OK\n");
+    return 0;
+}

tests/grib_clone_headers_only.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+# (C) Copyright 2005- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities granted to it by
+# virtue of its status as an intergovernmental organisation nor does it submit to any jurisdiction.
+#
+
+. ./include.ctest.sh
+
+label="grib_clone_headers_only_test"
+temp=temp.$label.grib
+
+inputs="
+  sst_globus0083.grib
+  sample.grib2
+  mixed.grib
+"
+
+for f in $inputs; do
+    infile=$data_dir/$f
+    rm -f $temp
+    $EXEC ${test_dir}/grib_clone_headers_only $infile $temp
+    ${tools_dir}/grib_compare -H -b totalLength $infile $temp
+done
+
+# Clean up
+rm -f $temp