From f3da045ba60f41343a509339f057e37312b0e1b9 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Mon, 15 Jan 2024 11:12:02 -0700 Subject: [PATCH 1/4] adding g2c_index test --- tests/run_index_tests.sh | 2 +- utils/g2c_index.c | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/run_index_tests.sh b/tests/run_index_tests.sh index 1429dd21..853b8405 100644 --- a/tests/run_index_tests.sh +++ b/tests/run_index_tests.sh @@ -10,7 +10,7 @@ echo "" echo "*** Running g2c_index test" # Read GRIB1 index. -../utils/g2c_index -v data/ref_gdaswave_2.grib1.idx g2c_index_gdaswave_2.grib1.idx.txt +../utils/g2c_index -v data/gdaswave.t00z.wcoast.0p16.f000.grib2 gdaswave.t00z.wcoast.0p16.f000.grib2.idx # Check against expected output. #diff -w g2c_degrib2_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2 data/ref_gdaswave.degrib2.txt diff --git a/utils/g2c_index.c b/utils/g2c_index.c index e1ccec6c..f26fbfde 100644 --- a/utils/g2c_index.c +++ b/utils/g2c_index.c @@ -30,8 +30,8 @@ main(int argc, char **argv) int index; int c; int p = 0; - /* int g2cid; */ - /* int ret; */ + int g2cid; + int ret; opterr = 0; @@ -69,16 +69,16 @@ main(int argc, char **argv) printf("g2c_index %s reading index file %s summarizing into %s.\n", G2C_VERSION, path[0], path[1]); /* Open the GRIB2 file. */ - /* if ((ret = g2c_open(path[0], G2C_NOWRITE, &g2cid))) */ - /* return ret; */ + if ((ret = g2c_open(path[0], G2C_NOWRITE, &g2cid))) + return ret; - /* /\* Write the degrib2 summary. *\/ */ - /* if ((ret = g2c_degrib2(g2cid, path[1]))) */ - /* return ret; */ + /* Write the index file. */ + if ((ret = g2c_write_index(g2cid, G2C_CLOBBER, path[1]))) + return ret; - /* /\* Close the file. *\/ */ - /* if ((ret = g2c_close(g2cid))) */ - /* return ret; */ + /* Close the file. */ + if ((ret = g2c_close(g2cid))) + return ret; /* Free memory. */ if (path[0]) From bf82a7140e6c485e0910ecf13b3d0d3bb2b5a84b Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Mon, 29 Jan 2024 05:59:53 -0700 Subject: [PATCH 2/4] changes --- src/g2cfile.c | 4 ++-- tests/run_index_tests.sh | 5 ++++- utils/g2c_degrib2.c | 7 +++++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/g2cfile.c b/src/g2cfile.c index 5b267cf6..ca80751e 100644 --- a/src/g2cfile.c +++ b/src/g2cfile.c @@ -1116,8 +1116,8 @@ read_metadata(int g2cid) LOG((6, "file_pos %ld", file_pos)); } - /* If we run out of messages, that's success. */ - if (ret == G2C_ENOMSG) + /* If we read some messages, then run out, that's success. */ + if (ret == G2C_ENOMSG && msg_num) ret = G2C_NOERROR; #ifdef LOGGING diff --git a/tests/run_index_tests.sh b/tests/run_index_tests.sh index 853b8405..4c87dd3d 100644 --- a/tests/run_index_tests.sh +++ b/tests/run_index_tests.sh @@ -9,9 +9,12 @@ set -e echo "" echo "*** Running g2c_index test" -# Read GRIB1 index. +# Create an index for a GRIB2 file. ../utils/g2c_index -v data/gdaswave.t00z.wcoast.0p16.f000.grib2 gdaswave.t00z.wcoast.0p16.f000.grib2.idx +# Summarize the index data. +../utils/g2c_degrib2 -v gdaswave.t00z.wcoast.0p16.f000.grib2.idx gdaswave.t00z.wcoast.0p16.f000.grib2.idx.degrib2 + # Check against expected output. #diff -w g2c_degrib2_gdaswave.t00z.wcoast.0p16.f000.grib2.degrib2 data/ref_gdaswave.degrib2.txt diff --git a/utils/g2c_degrib2.c b/utils/g2c_degrib2.c index 4d9749a0..0ced6287 100644 --- a/utils/g2c_degrib2.c +++ b/utils/g2c_degrib2.c @@ -62,6 +62,13 @@ main(int argc, char **argv) break; } + /* If we didn't get 2 files, error. */ + if (p != 2) + { + printf("Two filenames must be provided, for input and output.\n"); + return G2C_ERROR; + } + /* Yammer on and on. */ if (verbose) printf("g2c_degrib2 %s summarizing %s into %s.\n", G2C_VERSION, path[0], path[1]); From d9f9878fff730e5acafff9052840556760d0f53c Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Mon, 29 Jan 2024 06:43:39 -0700 Subject: [PATCH 3/4] dealing with large file index --- src/g2cindex.c | 106 +++++++++++++++++++++++++++++++++++++++++++--- src/grib2.h.in | 1 + utils/g2c_index.c | 13 ++++-- 3 files changed, 112 insertions(+), 8 deletions(-) diff --git a/src/g2cindex.c b/src/g2cindex.c index 09b1c966..4d92bb27 100644 --- a/src/g2cindex.c +++ b/src/g2cindex.c @@ -117,6 +117,87 @@ g2c_start_index_record(FILE *f, int rw_flag, int *reclen, int *msg, int *local, return G2C_NOERROR; } +/** + * Read or write the start of a version 2 index record for files > 2 + * GB. + * + * @param f FILE * to open index file. + * @param rw_flag True if function should write, false if it should read. + * @param reclen Pointer to reclen. + * @param msg Pointer to msg. + * @param local Pointer to local. + * @param gds Pointer to gds. + * @param pds Pointer to pds. + * @param drs Pointer to drs. + * @param bms Pointer to bms. + * @param data Pointer to data. + * @param msglen Pointer to msglen. + * @param version Pointer to version. + * @param discipline Pointer to discipline. + * @param fieldnum Pointer to fieldnum, 0- based. (It is 1-based in + * the index file.) + * + * @return + * - ::G2C_NOERROR No error. + * - ::G2C_EINVAL Invalid input. + * - ::G2C_EFILE File I/O error. + * + * @author Ed Hartnett 10/26/22 + */ +int +g2c_start_index_record_lf(FILE *f, int rw_flag, int *reclen, size_t *msg, int *local, int *gds, + int *pds, int *drs, int *bms, int *data, size_t *msglen, + unsigned char *version, unsigned char *discipline, short *fieldnum) +{ + /* size_t size_t_be; */ + short fieldnum1; /* This is for the 1-based fieldnum in the index file. */ + int ret; + + /* All pointers must be provided. */ + if (!f || !reclen || !msg || !local || !gds || !pds || !drs || !bms || !data + || !msglen || !version || !discipline || !fieldnum) + return G2C_EINVAL; + + /* When writing, set the fieldnum1 to be a 1-based index, just + * like in Fortran. */ + if (rw_flag) + fieldnum1 = *fieldnum + 1; + + /* Read or write the values at the beginning of each index + * record. */ + if ((ret = g2c_file_io_uint(f, rw_flag, (unsigned int *)reclen))) + return ret; + if ((ret = g2c_file_io_ulonglong(f, rw_flag, (unsigned long long *)msg))) + return ret; + if ((ret = g2c_file_io_uint(f, rw_flag, (unsigned int *)local))) + return ret; + if ((ret = g2c_file_io_uint(f, rw_flag, (unsigned int *)gds))) + return ret; + if ((ret = g2c_file_io_uint(f, rw_flag, (unsigned int *)pds))) + return ret; + if ((ret = g2c_file_io_uint(f, rw_flag, (unsigned int *)drs))) + return ret; + if ((ret = g2c_file_io_uint(f, rw_flag, (unsigned int *)bms))) + return ret; + if ((ret = g2c_file_io_uint(f, rw_flag, (unsigned int *)data))) + return ret; + if ((ret = g2c_file_io_ulonglong(f, rw_flag, (unsigned long long *)msglen))) + return ret; + if ((ret = g2c_file_io_ubyte(f, rw_flag, version))) + return ret; + if ((ret = g2c_file_io_ubyte(f, rw_flag, discipline))) + return ret; + if ((ret = g2c_file_io_short(f, rw_flag, &fieldnum1))) + return ret; + + /* When reading, translate the 1-based fieldnum1 into the 0-based + * fieldnum that C programmers will expect and love. */ + if (!rw_flag) + *fieldnum = fieldnum1 - 1; + + return G2C_NOERROR; +} + /** * Read or write the start of a version 1 index record. * @@ -354,6 +435,7 @@ g2c_write_index(int g2cid, int mode, const char *index_file) char my_path[G2C_INDEX_BASENAME_LEN + 1]; G2C_MESSAGE_INFO_T *msg; int total_index_size = 0; /* Does not include size of header records. */ + int large_file_index = 0; /* True if indexed file may be > 2 GB. */ int reclen; int ret = G2C_NOERROR; @@ -377,6 +459,10 @@ g2c_write_index(int g2cid, int mode, const char *index_file) } } + /* If LARGE_INDEX_FILE, check if file exists. */ + if (mode & G2C_LARGE_INDEX_FILE) + large_index_file++; + /* Create the index file. */ if (!(f = fopen(index_file, "wb+"))) return G2C_EFILE; @@ -451,7 +537,6 @@ g2c_write_index(int g2cid, int mode, const char *index_file) for (fieldnum = 0; fieldnum < msg->num_fields; fieldnum++) { G2C_SECTION_INFO_T *sec3, *sec4, *sec5, *sec6, *sec7; - int bytes_to_msg = (int)msg->bytes_to_msg; int bs3, bs4, bs5, bs6, bs7; /* bytes to each section, as ints. */ unsigned char sec_num; int ret; @@ -470,10 +555,21 @@ g2c_write_index(int g2cid, int mode, const char *index_file) LOG((4, "fieldnum %d reclen %d", fieldnum, reclen)); /* Write the beginning of the index record. */ - if ((ret = g2c_start_index_record(f, G2C_FILE_WRITE, &reclen, &bytes_to_msg, &msg->bytes_to_local, - &bs3, &bs4, &bs5, &bs6, &bs7, &msg->bytes_in_msg, &msg->master_version, - &msg->discipline, &fieldnum))) - break; + if (large_file_index) + { + int bytes_to_msg = (int)msg->bytes_to_msg; + if ((ret = g2c_start_index_record(f, G2C_FILE_WRITE, &reclen, &bytes_to_msg, &msg->bytes_to_local, + &bs3, &bs4, &bs5, &bs6, &bs7, &msg->bytes_in_msg, &msg->master_version, + &msg->discipline, &fieldnum))) + break; + } + else + { + if ((ret = g2c_start_index_record_lf(f, G2C_FILE_WRITE, &reclen, &msg->bytes_to_msg, &msg->bytes_to_local, + &bs3, &bs4, &bs5, &bs6, &bs7, &msg->bytes_in_msg, &msg->master_version, + &msg->discipline, &fieldnum))) + break; + } /* Write the section 1, identification section. */ if ((ret = g2c_rw_section1_metadata(f, G2C_FILE_WRITE, msg))) diff --git a/src/grib2.h.in b/src/grib2.h.in index 8faf92b8..3cfa0e1b 100644 --- a/src/grib2.h.in +++ b/src/grib2.h.in @@ -291,6 +291,7 @@ g2int aecunpack(unsigned char *cpack, g2int len, g2int *idrstmpl, g2int ndpts, #define G2C_WRITE 0x0001 /**< Set read-write access for g2c_open(). */ #define G2C_CLOBBER 0x0000 /**< Destroy existing file. Mode flag for g2c_create(). */ #define G2C_NOCLOBBER 0x0004 /**< Don't destroy existing file. Mode flag for g2c_create(). */ +#define G2C_LARGE_FILE_INDEX 0x0008 /**< Create a large file index. Mode flag for g2c_write_index(). */ /* Useful constants. */ #define G2C_SECTION0_BYTES 16 /**< Number of bytes in section 0. */ diff --git a/utils/g2c_index.c b/utils/g2c_index.c index f26fbfde..7644a0c9 100644 --- a/utils/g2c_index.c +++ b/utils/g2c_index.c @@ -1,6 +1,6 @@ /** @file * - * @brief Print a summary of a GRIB1 or GRIB2 index file. + * @brief Write a GRIB2 index file. * * @author Ed Hartnett @date 12/29/22 */ @@ -11,7 +11,7 @@ #include /** - * Print a summary of a GRIB1 or GRIB2 index file. + * Write a GRIB2 index file. * * @param argc Number of arguments. * @param argv Pointer to array of arguments. @@ -27,7 +27,9 @@ main(int argc, char **argv) { char *path[2] = {NULL, NULL}; int verbose = 0; + int large_file_index = 0; int index; + int write_index_flag = G2C_CLOBBER; int c; int p = 0; int g2cid; @@ -43,6 +45,9 @@ main(int argc, char **argv) case 'v': verbose = 1; break; + case 'l': + large_file_index = 1; + break; case '?': if (isprint(optopt)) fprintf(stderr, "Unknown option `-%c'.\n", optopt); @@ -73,7 +78,9 @@ main(int argc, char **argv) return ret; /* Write the index file. */ - if ((ret = g2c_write_index(g2cid, G2C_CLOBBER, path[1]))) + if (large_file_index) + write_index_flag & G2C_LARGE_FILE_INDEX; + if ((ret = g2c_write_index(g2cid, write_index_flag, path[1]))) return ret; /* Close the file. */ From dffb0e4b98369cbff4623ffe7940f4c0092170f0 Mon Sep 17 00:00:00 2001 From: Edward Hartnett Date: Mon, 29 Jan 2024 07:59:00 -0700 Subject: [PATCH 4/4] fixed --- src/g2cindex.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/g2cindex.c b/src/g2cindex.c index 4d92bb27..ffd52c0a 100644 --- a/src/g2cindex.c +++ b/src/g2cindex.c @@ -460,7 +460,7 @@ g2c_write_index(int g2cid, int mode, const char *index_file) } /* If LARGE_INDEX_FILE, check if file exists. */ - if (mode & G2C_LARGE_INDEX_FILE) + if (mode & G2C_LARGE_FILE_INDEX) large_index_file++; /* Create the index file. */