From a5437b4473d90d4cf4e4d2f27f0b2c023364c8a7 Mon Sep 17 00:00:00 2001 From: Thomas Coleman <15375218+ColemanTom@users.noreply.github.com> Date: Wed, 17 Jun 2026 13:26:59 +1000 Subject: [PATCH] Add functionality to index grib files as one is made Previous workflow: - create grib file - save it - create index from that grib file New workflow: - create message for grib file - save message - add message details to index file - repeat for remaining messages in file There is no need for double handling of the grib file data anymore, providing significant time savings for this pattern. --- fortran/eccodes_f90_tail.f90 | 34 +++ fortran/grib_api_externals.h | 2 + fortran/grib_f90_tail.f90 | 53 ++++ fortran/grib_fortran.cc | 27 ++- fortran/grib_fortran_prototypes.h | 6 + src/eccodes/eccodes.h | 29 +++ src/eccodes/grib_api.h | 27 +++ src/eccodes/grib_index.cc | 386 ++++++++++++++++++------------ 8 files changed, 409 insertions(+), 155 deletions(-) diff --git a/fortran/eccodes_f90_tail.f90 b/fortran/eccodes_f90_tail.f90 index 537d168cae..021f6dac88 100644 --- a/fortran/eccodes_f90_tail.f90 +++ b/fortran/eccodes_f90_tail.f90 @@ -50,6 +50,22 @@ subroutine codes_index_create(indexid, filename, keys, status) call grib_index_create(indexid, filename, keys, status) end subroutine codes_index_create + !> Create a new empty index with the given keys. + !> + !> Unlike codes_index_create, no file is scanned. Messages can be added + !> later with codes_index_add_message. + !> + !> @param indexid ID of the newly created index + !> @param keys comma separated list of keys for the index + !> @param status CODES_SUCCESS if OK, integer value on error + subroutine codes_index_new(indexid, keys, status) + integer(kind=kindOfInt), intent(inout) :: indexid + character(len=*), intent(in) :: keys + integer(kind=kindOfInt), optional, intent(out) :: status + + call grib_index_new(indexid, keys, status) + end subroutine codes_index_new + !> Add a file to an index. !> !> In case of error, if the status parameter (optional) is not given, the program will @@ -69,6 +85,24 @@ subroutine codes_index_add_file(indexid, filename, status) call grib_index_add_file(indexid, filename, status) end subroutine codes_index_add_file + !> Add a GRIB message handle to an index at the given file offset. + !> See grib_index_add_message for full documentation. + !> + !> @param indexid ID of the index to update + !> @param gribid ID of the message handle to index + !> @param filename name of the file containing the message + !> @param offset byte offset of the message within the file + !> @param status CODES_SUCCESS if OK, integer value on error + subroutine codes_index_add_message(indexid, gribid, filename, offset, status) + integer(kind=kindOfInt), intent(in) :: indexid + integer(kind=kindOfInt), intent(in) :: gribid + character(len=*), intent(in) :: filename + integer(kind=8), intent(in) :: offset + integer(kind=kindOfInt), optional, intent(out) :: status + + call grib_index_add_message(indexid, gribid, filename, offset, status) + end subroutine codes_index_add_message + !> Get the number of distinct values of the key in argument contained in the index. The key must belong to the index. !> !> In case of error, if the status parameter (optional) is not given, the program will diff --git a/fortran/grib_api_externals.h b/fortran/grib_api_externals.h index 67e4a7698f..b06505e4cc 100644 --- a/fortran/grib_api_externals.h +++ b/fortran/grib_api_externals.h @@ -54,7 +54,9 @@ integer, external :: grib_f_get_int, grib_f_get_long,grib_f_get_int_array, & grib_f_grib_surface_type_requires_value integer, external :: grib_f_new_from_index, & grib_f_index_new_from_file, & + grib_f_index_new, & grib_f_index_add_file, & + grib_f_index_add_message, & grib_f_index_read, & grib_f_index_write, & grib_f_index_release, & diff --git a/fortran/grib_f90_tail.f90 b/fortran/grib_f90_tail.f90 index 4c13f31841..b51905ebcb 100644 --- a/fortran/grib_f90_tail.f90 +++ b/fortran/grib_f90_tail.f90 @@ -60,6 +60,28 @@ subroutine grib_index_create(indexid, filename, keys, status) end if end subroutine grib_index_create + !> Create a new empty index with the given keys. + !> + !> Unlike grib_index_create, no file is scanned. Messages can be added + !> later with grib_index_add_message. + !> + !> @param indexid ID of the newly created index + !> @param keys comma separated list of keys for the index + !> @param status GRIB_SUCCESS if OK, integer value on error + subroutine grib_index_new(indexid, keys, status) + integer(kind=kindOfInt), intent(inout) :: indexid + character(len=*), intent(in) :: keys + integer(kind=kindOfInt), optional, intent(out) :: status + integer(kind=kindOfInt) :: iret + + iret = grib_f_index_new(indexid, keys) + if (present(status)) then + status = iret + else + call grib_check(iret, 'index_new', '('//keys//')') + end if + end subroutine grib_index_new + !> Add a file to an index. !> !> @@ -86,6 +108,37 @@ subroutine grib_index_add_file(indexid, filename, status) end if end subroutine grib_index_add_file + !> Add a message handle to an index at the given file offset. + !> + !> The message must already have been written to the file at 'offset'. + !> Typical usage: record ftell() before writing, write the message, + !> then call this to update the index without re-scanning the file. + !> + !> In case of error, if the status parameter (optional) is not given, the program will + !> exit with an error message.\n Otherwise the error message can be + !> gathered with @ref grib_get_error_string. + !> + !> @param indexid ID of the index to update + !> @param gribid ID of the GRIB message handle to index + !> @param filename name of the file containing the message + !> @param offset byte offset of the message within the file + !> @param status GRIB_SUCCESS if OK, integer value on error + subroutine grib_index_add_message(indexid, gribid, filename, offset, status) + integer(kind=kindOfInt), intent(in) :: indexid + integer(kind=kindOfInt), intent(in) :: gribid + character(len=*), intent(in) :: filename + integer(kind=8), intent(in) :: offset + integer(kind=kindOfInt), optional, intent(out) :: status + integer(kind=kindOfInt) :: iret + + iret = grib_f_index_add_message(indexid, gribid, filename, offset) + if (present(status)) then + status = iret + else + call grib_check(iret, 'index_add_message', '('//filename//')') + end if + end subroutine grib_index_add_message + !> Get the number of distinct values of the key in argument contained in the index. The key must belong to the index. !> !> diff --git a/fortran/grib_fortran.cc b/fortran/grib_fortran.cc index 301678efdb..7b542073f8 100644 --- a/fortran/grib_fortran.cc +++ b/fortran/grib_fortran.cc @@ -1725,6 +1725,31 @@ int grib_f_index_add_file_(int* index_id, char* file, int lfile) } } +/*****************************************************************************/ +int grib_f_index_add_message_(int* index_id, int* grib_id, char* file, int64_t* offset, int lfile) +{ + grib_index* idx = get_index(*index_id); + if (!idx) return GRIB_INVALID_INDEX; + + grib_handle* h = get_handle(*grib_id); + if (!h) return GRIB_INVALID_GRIB; + + char buf[1024]; + + return codes_index_add_message(idx, h, cast_char(buf, file, lfile), (off_t)*offset); +} + +/*****************************************************************************/ +int grib_f_index_new_(int* gid, char* keys, int lkeys) +{ + char buf[1024]; + int err = 0; + grib_index* i = grib_index_new(grib_context_get_default(), + cast_char(buf, keys, lkeys), &err); + if (err) { *gid = -1; return err; } + push_index(i, gid); + return GRIB_SUCCESS; +} /*****************************************************************************/ int grib_f_index_read_(char* file, int* gid, int lfile) { @@ -2706,7 +2731,7 @@ int grib_f_set_string_(int* gid, char* key, char* val, int len, int len2) size_t lsize = len2; if(!h) return GRIB_INVALID_GRIB; - + /* For BUFR, the value may contain spaces e.g. stationOrSiteName='CAMPO NOVO' */ /* So do not use cast_char. cast_char_no_cut does not stop at first space */ val_str = cast_char_no_cut(buf2,val,len2); diff --git a/fortran/grib_fortran_prototypes.h b/fortran/grib_fortran_prototypes.h index cddc58401e..c817123118 100644 --- a/fortran/grib_fortran_prototypes.h +++ b/fortran/grib_fortran_prototypes.h @@ -128,9 +128,15 @@ int grib_f_new_from_index(int *iid, int *gid); int grib_f_index_new_from_file_(char *file, char *keys, int *gid, int lfile, int lkeys); int grib_f_index_new_from_file__(char *file, char *keys, int *gid, int lfile, int lkeys); int grib_f_index_new_from_file(char *file, char *keys, int *gid, int lfile, int lkeys); +int grib_f_index_new_(int* gid, char* keys, int lkeys); +int grib_f_index_new__(int* gid, char* keys, int lkeys); +int grib_f_index_new(int* gid, char* keys, int lkeys); int grib_f_index_add_file_(int* iid, char* file, int lfile); int grib_f_index_add_file__(int* iid, char* file, int lfile); int grib_f_index_add_file(int* iid, char* file, int lfile); +int grib_f_index_add_message_(int* index_id, int* grib_id, char* file, int64_t* offset, int lfile); +int grib_f_index_add_message__(int* index_id, int* grib_id, char* file, int64_t* offset, int lfile); +int grib_f_index_add_message(int* index_id, int* grib_id, char* file, int64_t* offset, int lfile); int grib_f_index_read_(char *file, int *gid, int lfile); int grib_f_index_read__(char *file, int *gid, int lfile); int grib_f_index_read(char *file, int *gid, int lfile); diff --git a/src/eccodes/eccodes.h b/src/eccodes/eccodes.h index 86a2531fba..a83ea8530a 100644 --- a/src/eccodes/eccodes.h +++ b/src/eccodes/eccodes.h @@ -236,6 +236,35 @@ codes_index* codes_index_new(codes_context* c, const char* keys, int* err); * @return 0 if OK, integer value on error */ int codes_index_add_file(codes_index* index, const char* filename); + +/** + * Index one message handle into the index at the given file offset. + * + * All key extraction is performed from the in-memory handle 'h'; the file + * content is NOT read by this function. The file must already exist and be + * openable so its name can be registered in the index, but it does not need + * to contain the message yet. + * + * Typical usage: + * offset = ftell(fp); + * codes_write_message(h, fp); // write the message + * codes_index_add_message(idx, h, path, offset); // index immediately + * // ... continue writing more messages ... + * // Flush before reading back through the index: + * fflush(fp); + * codes_handle_new_from_index(idx, &err); + * + * The handle's product kind must match the index product kind. + * + * @param index : index + * @param h : handle of the message to index + * @param filename : path of the file the message is being written to (must exist) + * @param offset : byte offset of the message within the file + * @return 0 if OK, integer value on error + */ +int codes_index_add_message(codes_index* index, codes_handle* h, + const char* filename, off_t offset); + int codes_index_write(codes_index* index, const char* filename); codes_index* codes_index_read(codes_context* c, const char* filename, int* err); diff --git a/src/eccodes/grib_api.h b/src/eccodes/grib_api.h index 0bb03f3947..eb4b65c0c6 100644 --- a/src/eccodes/grib_api.h +++ b/src/eccodes/grib_api.h @@ -287,6 +287,33 @@ int grib_index_add_file(grib_index* index, const char* filename); int grib_index_write(grib_index* index, const char* filename); grib_index* grib_index_read(grib_context* c, const char* filename, int* err); +/** + * Index one message handle into the index at the given file offset. + * + * All key extraction is performed from the in-memory handle 'h'; the file + * content is NOT read by this function. The file must already exist and be + * openable so its name can be registered in the index, but it does not need + * to contain the message yet. + * + * Typical usage: + * offset = ftell(fp); + * grib_write_message(h, fp); // write the message + * codes_index_add_message(idx, h, path, offset); // index immediately + * // ... continue writing more messages ... + * // Flush before reading back through the index: + * fflush(fp); + * grib_handle_new_from_index(idx, &err); + * + * The handle's product kind must match the index product kind. + * + * @param index : index + * @param h : handle of the message to index + * @param filename : path of the file the message is being written to (must exist) + * @param offset : byte offset of the message within the file + * @return 0 if OK, integer value on error + */ +int codes_index_add_message(grib_index* index, grib_handle* h, const char* filename, off_t offset); + /** * Get the number of distinct values of the key in argument contained in the index. The key must belong to the index. * diff --git a/src/eccodes/grib_index.cc b/src/eccodes/grib_index.cc index 999441bf0a..5f3a0d2be6 100644 --- a/src/eccodes/grib_index.cc +++ b/src/eccodes/grib_index.cc @@ -1072,20 +1072,166 @@ static grib_handle* new_message_from_file(int message_type, grib_context* c, FIL #define MAX_NUM_KEYS 40 -static int codes_index_add_file_internal(grib_index* index, const char* filename, int message_type) +/* Internal helper: index one message into the tree. + * file must already be registered in index->files. + * offset and length are the message's position in the file. */ +static int codes_index_add_message_to_file_internal(grib_index* index, grib_handle* h, + grib_file* file, off_t offset, long length) { double dval; size_t svallen; - size_t message_count = 0; - long length, lval; + long lval; char buf[1024] = {0,}; int err = 0; + grib_string_list* v = 0; + grib_index_key* index_key = index->keys; + grib_field_tree* field_tree = index->fields; + grib_field* field; + grib_context* c = index->context; + + index_key->value[0] = 0; + + { + const char* set_keys_env_var = "ECCODES_INDEX_SET_KEYS"; + char* envsetkeys = getenv(set_keys_env_var); + if (envsetkeys) { + grib_values set_values[MAX_NUM_KEYS]; + int set_values_count = MAX_NUM_KEYS; + std::string copy_of_env(envsetkeys); /* parse_keyval_string modifies envsetkeys! */ + int error = parse_keyval_string(NULL, envsetkeys, 1, GRIB_TYPE_UNDEFINED, + set_values, &set_values_count); + if (!error && set_values_count != 0) { + err = grib_set_values(h, set_values, set_values_count); + if (err) { + grib_context_log(c, GRIB_LOG_ERROR, + "codes_index_add_message: Unable to set %s", copy_of_env.c_str()); + return err; + } + } else { + grib_context_log(c, GRIB_LOG_ERROR, "codes_index_add_message: Unable to parse %s (%s)", + set_keys_env_var, grib_get_error_message(error)); + return error; + } + } + } + + if (index->product_kind == PRODUCT_BUFR && index->unpack_bufr) { + err = grib_set_long(h, "unpack", 1); + if (err) { + grib_context_log(c, GRIB_LOG_ERROR, "Unable to unpack BUFR to create index. \"%s\": %s", + index_key->name, grib_get_error_message(err)); + return err; + } + } + + while (index_key) { + if (index_key->type == GRIB_TYPE_UNDEFINED) { + err = grib_get_native_type(h, index_key->name, &(index_key->type)); + if (err) + index_key->type = GRIB_TYPE_STRING; + } + svallen = 1024; + switch (index_key->type) { + case GRIB_TYPE_STRING: + err = grib_get_string(h, index_key->name, buf, &svallen); + if (err == GRIB_NOT_FOUND) + snprintf(buf, sizeof(buf), GRIB_KEY_UNDEF); + break; + case GRIB_TYPE_LONG: + err = grib_get_long(h, index_key->name, &lval); + if (err == GRIB_NOT_FOUND) + snprintf(buf, sizeof(buf), GRIB_KEY_UNDEF); + else + snprintf(buf, sizeof(buf), "%ld", lval); + break; + case GRIB_TYPE_DOUBLE: + err = grib_get_double(h, index_key->name, &dval); + if (err == GRIB_NOT_FOUND) + snprintf(buf, sizeof(buf), GRIB_KEY_UNDEF); + else + snprintf(buf, sizeof(buf), "%g", dval); + break; + default: + err = GRIB_WRONG_TYPE; + return err; + } + if (err && err != GRIB_NOT_FOUND) { + grib_context_log(c, GRIB_LOG_ERROR, "Unable to index message. key=\"%s\": %s", + index_key->name, grib_get_error_message(err)); + return err; + } + + if (!index_key->values->value) { + index_key->values->value = grib_context_strdup(c, buf); + index_key->values_count++; + } + else { + v = index_key->values; + while (v->next && strcmp(v->value, buf)) + v = v->next; + if (strcmp(v->value, buf)) { + index_key->values_count++; + if (v->next) + v = v->next; + v->next = (grib_string_list*)grib_context_malloc_clear(c, sizeof(grib_string_list)); + v->next->value = grib_context_strdup(c, buf); + } + } + + if (!field_tree->value) { + field_tree->value = grib_context_strdup(c, buf); + } + else { + while (field_tree->next && + (field_tree->value == NULL || + strcmp(field_tree->value, buf))) + field_tree = field_tree->next; + + if (!field_tree->value || strcmp(field_tree->value, buf)) { + field_tree->next = + (grib_field_tree*)grib_context_malloc_clear(c, sizeof(grib_field_tree)); + field_tree = field_tree->next; + field_tree->value = grib_context_strdup(c, buf); + } + } + + if (index_key->next) { + if (!field_tree->next_level) { + field_tree->next_level = + (grib_field_tree*)grib_context_malloc_clear(c, sizeof(grib_field_tree)); + } + field_tree = field_tree->next_level; + } + index_key = index_key->next; + } + + field = (grib_field*)grib_context_malloc_clear(c, sizeof(grib_field)); + field->file = file; + field->offset = offset; + field->length = length; + index->count++; + + if (field_tree->field) { + grib_field* pfield = field_tree->field; + while (pfield->next) + pfield = pfield->next; + pfield->next = field; + } + else + field_tree->field = field; + + index->rewind = 1; + return GRIB_SUCCESS; +} + +static int codes_index_add_file_internal(grib_index* index, const char* filename, int message_type) +{ + size_t message_count = 0; + long length; + int err = 0; grib_file* indfile; - grib_index_key* index_key = NULL; - grib_handle* h = NULL; - grib_field* field; - grib_field_tree* field_tree; + grib_handle* h = NULL; grib_file* file = NULL; grib_context* c; bool warn_about_duplicates = true; @@ -1121,166 +1267,24 @@ static int codes_index_add_file_internal(grib_index* index, const char* filename std::map map_of_offsets; while ((h = new_message_from_file(message_type, c, file->handle, &err)) != NULL) { - grib_string_list* v = 0; - index_key = index->keys; - field_tree = index->fields; - index_key->value[0] = 0; message_count++; - { - const char* set_keys_env_var = "ECCODES_INDEX_SET_KEYS"; - char* envsetkeys = getenv(set_keys_env_var); - if (envsetkeys) { - grib_values set_values[MAX_NUM_KEYS]; - int set_values_count = MAX_NUM_KEYS; - std::string copy_of_env(envsetkeys); //parse_keyval_string changes envsetkeys! - int error = parse_keyval_string(NULL, envsetkeys, 1, GRIB_TYPE_UNDEFINED, - set_values, &set_values_count); - if (!error && set_values_count != 0) { - err = grib_set_values(h, set_values, set_values_count); - if (err) { - grib_context_log(c, GRIB_LOG_ERROR, - "codes_index_add_file: Unable to set %s", copy_of_env.c_str()); - return err; - } - } else { - grib_context_log(c, GRIB_LOG_ERROR, "codes_index_add_file: Unable to parse %s (%s)", - set_keys_env_var, grib_get_error_message(error)); - return error; - } - } - } - - if (index->product_kind == PRODUCT_BUFR && index->unpack_bufr) { - err = grib_set_long(h, "unpack", 1); - if (err) { - grib_context_log(c, GRIB_LOG_ERROR, "Unable to unpack BUFR to create index. \"%s\": %s", - index_key->name, grib_get_error_message(err)); - return err; - } - } - - while (index_key) { - if (index_key->type == GRIB_TYPE_UNDEFINED) { - err = grib_get_native_type(h, index_key->name, &(index_key->type)); - if (err) - index_key->type = GRIB_TYPE_STRING; - } - svallen = 1024; - switch (index_key->type) { - case GRIB_TYPE_STRING: - err = grib_get_string(h, index_key->name, buf, &svallen); - if (err == GRIB_NOT_FOUND) - snprintf(buf, sizeof(buf), GRIB_KEY_UNDEF); - break; - case GRIB_TYPE_LONG: - err = grib_get_long(h, index_key->name, &lval); - if (err == GRIB_NOT_FOUND) - snprintf(buf, sizeof(buf), GRIB_KEY_UNDEF); - else - snprintf(buf, sizeof(buf), "%ld", lval); - break; - case GRIB_TYPE_DOUBLE: - err = grib_get_double(h, index_key->name, &dval); - if (err == GRIB_NOT_FOUND) - snprintf(buf, sizeof(buf), GRIB_KEY_UNDEF); - else - snprintf(buf, sizeof(buf), "%g", dval); - break; - default: - err = GRIB_WRONG_TYPE; - return err; - } - if (err && err != GRIB_NOT_FOUND) { - grib_context_log(c, GRIB_LOG_ERROR, "Unable to create index. key=\"%s\" (message #%lu): %s", - index_key->name, message_count, grib_get_error_message(err)); - return err; - } - - if (!index_key->values->value) { - index_key->values->value = grib_context_strdup(c, buf); - index_key->values_count++; - } - else { - v = index_key->values; - while (v->next && strcmp(v->value, buf)) - v = v->next; - if (strcmp(v->value, buf)) { - index_key->values_count++; - if (v->next) - v = v->next; - v->next = (grib_string_list*)grib_context_malloc_clear(c, sizeof(grib_string_list)); - v->next->value = grib_context_strdup(c, buf); - } - } - - if (!field_tree->value) { - field_tree->value = grib_context_strdup(c, buf); - } - else { - while (field_tree->next && - (field_tree->value == NULL || - strcmp(field_tree->value, buf))) - field_tree = field_tree->next; - - if (!field_tree->value || strcmp(field_tree->value, buf)) { - field_tree->next = - (grib_field_tree*)grib_context_malloc_clear(c, - sizeof(grib_field_tree)); - field_tree = field_tree->next; - field_tree->value = grib_context_strdup(c, buf); - } - } - - if (index_key->next) { - if (!field_tree->next_level) { - field_tree->next_level = - (grib_field_tree*)grib_context_malloc_clear(c, sizeof(grib_field_tree)); - } - field_tree = field_tree->next_level; - } - index_key = index_key->next; - } - - field = (grib_field*)grib_context_malloc_clear(c, sizeof(grib_field)); - field->file = file; - index->count++; - field->offset = h->offset; - if (warn_about_duplicates) { - const bool offset_is_unique = map_of_offsets.insert( std::pair(h->offset, h) ).second; - if (!offset_is_unique) { - fprintf(stderr, "ECCODES WARNING : File '%s': field offset %ld is not unique.\n", filename, (long)h->offset); - long edition = 0; - if (grib_get_long(h, "edition", &edition) == GRIB_SUCCESS && edition == 2) { - fprintf(stderr, "ECCODES WARNING : This can happen if the file contains multi-field GRIB messages.\n"); - fprintf(stderr, "ECCODES WARNING : Indexing multi-field messages is not fully supported.\n"); - } - warn_about_duplicates = false; - } - } - err = grib_get_long(h, "totalLength", &length); - if (err) + if (err) { + grib_handle_delete(h); return err; - field->length = length; - - if (field_tree->field) { - grib_field* pfield = field_tree->field; - while (pfield->next) - pfield = pfield->next; - pfield->next = field; } - else - field_tree->field = field; + err = codes_index_add_message_to_file_internal(index, h, file, h->offset, length); grib_handle_delete(h); + if (err) + return err; }/*foreach message*/ grib_file_close(file->name, 0, &err); if (err) return err; - index->rewind = 1; if (message_count == 0) { grib_context_log(c, GRIB_LOG_ERROR, "File %s contains no messages", filename); return GRIB_END_OF_FILE; @@ -1293,6 +1297,80 @@ static int codes_index_add_file_internal(grib_index* index, const char* filename return GRIB_SUCCESS; } +/* Public API: Index one message handle into the index at the given file offset. + * The file at 'filename' must already contain the message at 'offset'. + * offset is the byte offset of the message within the file. */ +static int codes_index_add_message_internal(grib_index* index, grib_handle* h, + const char* filename, off_t offset) +{ + int err = 0; + long length = 0; + grib_file* pool_file = NULL; + grib_file* indfile = NULL; + grib_context* c; + + if (!index) return GRIB_NULL_INDEX; + if (!h || !filename) return GRIB_INVALID_ARGUMENT; + c = index->context; + + /* Validate that the handle's product kind matches the index */ + if (h->product_kind != index->product_kind) { + grib_context_log(c, GRIB_LOG_ERROR, + "codes_index_add_message: handle product kind does not match index"); + return GRIB_INVALID_ARGUMENT; + } + + err = grib_get_long(h, "totalLength", &length); + if (err) return err; + + /* If this filename is already registered in the index, no file I/O is needed. + * All key extraction comes from the in-memory handle; the pool entry's name + * is all that is needed for later readback. */ + indfile = index->files; + while (indfile) { + if (!strcmp(indfile->name, filename)) { + /* Clone's pool_file points back to the real pool entry */ + return codes_index_add_message_to_file_internal( + index, h, indfile->pool_file, offset, length); + } + indfile = indfile->next; + } + + /* First message for this filename: open briefly to register it in the file + * pool so the pool assigns an id and keeps the name. No bytes are read. */ + pool_file = grib_file_open(filename, "r", &err); + if (!pool_file || !pool_file->handle) + return err ? err : GRIB_IO_PROBLEM; + + /* Append a clone to index->files */ + grib_filesid++; + if (!index->files) { + index->files = grib_file_pool_create_clone(c, grib_filesid, pool_file); + } + else { + indfile = index->files; + while (indfile->next) + indfile = indfile->next; + indfile->next = grib_file_pool_create_clone(c, grib_filesid, pool_file); + } + + err = codes_index_add_message_to_file_internal(index, h, pool_file, offset, length); + { + int close_err = 0; + grib_file_close(filename, 0, &close_err); + if (!err) err = close_err; + } + return err; +} + +// C-API: Ensure all exceptions are converted to error codes +int codes_index_add_message(grib_index* index, grib_handle* h, + const char* filename, off_t offset) +{ + auto result = eccodes::handleExceptions(codes_index_add_message_internal, index, h, filename, offset); + return eccodes::getErrorCode(result); +} + // int grib_index_add_file(grib_index* index, const char* filename) // { // double dval;