Add functionality to index grib files as a grib file is written

fortran/eccodes_f90_tail.f90

@@ -50,6 +50,22 @@ subroutine codes_index_create(indexid, filename, keys, status)
     call grib_index_create(indexid, filename, keys, status)
   end subroutine codes_index_create
 
+  !> Create a new empty index with the given keys.
+  !>
+  !> Unlike codes_index_create, no file is scanned. Messages can be added
+  !> later with codes_index_add_message.
+  !>
+  !> @param indexid     ID of the newly created index
+  !> @param keys        comma separated list of keys for the index
+  !> @param status      CODES_SUCCESS if OK, integer value on error
+  subroutine codes_index_new(indexid, keys, status)
+    integer(kind=kindOfInt), intent(inout)          :: indexid
+    character(len=*), intent(in)                    :: keys
+    integer(kind=kindOfInt), optional, intent(out)  :: status
+
+    call grib_index_new(indexid, keys, status)
+  end subroutine codes_index_new
+
   !> Add a file to an index.
   !>
   !> In case of error, if the status parameter (optional) is not given, the program will
@@ -69,6 +85,24 @@ subroutine codes_index_add_file(indexid, filename, status)
     call grib_index_add_file(indexid, filename, status)
   end subroutine codes_index_add_file
 
+  !> Add a GRIB message handle to an index at the given file offset.
+  !> See grib_index_add_message for full documentation.
+  !>
+  !> @param indexid     ID of the index to update
+  !> @param gribid      ID of the message handle to index
+  !> @param filename    name of the file containing the message
+  !> @param offset      byte offset of the message within the file
+  !> @param status      CODES_SUCCESS if OK, integer value on error
+  subroutine codes_index_add_message(indexid, gribid, filename, offset, status)
+    integer(kind=kindOfInt), intent(in)             :: indexid
+    integer(kind=kindOfInt), intent(in)             :: gribid
+    character(len=*), intent(in)                    :: filename
+    integer(kind=8), intent(in)                     :: offset
+    integer(kind=kindOfInt), optional, intent(out)  :: status
+
+    call grib_index_add_message(indexid, gribid, filename, offset, status)
+  end subroutine codes_index_add_message
+
   !> Get the number of distinct values of the key in argument contained in the index. The key must belong to the index.
   !>
   !> In case of error, if the status parameter (optional) is not given, the program will

fortran/grib_api_externals.h

@@ -54,7 +54,9 @@ integer, external :: grib_f_get_int, grib_f_get_long,grib_f_get_int_array, &
                      grib_f_grib_surface_type_requires_value
 integer, external :: grib_f_new_from_index, &
                      grib_f_index_new_from_file, &
+                     grib_f_index_new, &
                      grib_f_index_add_file, &
+                     grib_f_index_add_message, &
                      grib_f_index_read, &
                      grib_f_index_write, &
                      grib_f_index_release, &

fortran/grib_f90_tail.f90

@@ -60,6 +60,28 @@ subroutine grib_index_create(indexid, filename, keys, status)
     end if
   end subroutine grib_index_create
 
+  !> Create a new empty index with the given keys.
+  !>
+  !> Unlike grib_index_create, no file is scanned. Messages can be added
+  !> later with grib_index_add_message.
+  !>
+  !> @param indexid     ID of the newly created index
+  !> @param keys        comma separated list of keys for the index
+  !> @param status      GRIB_SUCCESS if OK, integer value on error
+  subroutine grib_index_new(indexid, keys, status)
+    integer(kind=kindOfInt), intent(inout)          :: indexid
+    character(len=*), intent(in)                    :: keys
+    integer(kind=kindOfInt), optional, intent(out)  :: status
+    integer(kind=kindOfInt)                         :: iret
+
+    iret = grib_f_index_new(indexid, keys)
+    if (present(status)) then
+      status = iret
+    else
+      call grib_check(iret, 'index_new', '('//keys//')')
+    end if
+  end subroutine grib_index_new
+
   !> Add a file to an index.
   !>
   !>
@@ -86,6 +108,37 @@ subroutine grib_index_add_file(indexid, filename, status)
     end if
   end subroutine grib_index_add_file
 
+  !> Add a message handle to an index at the given file offset.
+  !>
+  !> The message must already have been written to the file at 'offset'.
+  !> Typical usage: record ftell() before writing, write the message,
+  !> then call this to update the index without re-scanning the file.
+  !>
+  !> In case of error, if the status parameter (optional) is not given, the program will
+  !> exit with an error message.\n Otherwise the error message can be
+  !> gathered with @ref grib_get_error_string.
+  !>
+  !> @param indexid     ID of the index to update
+  !> @param gribid      ID of the GRIB message handle to index
+  !> @param filename    name of the file containing the message
+  !> @param offset      byte offset of the message within the file
+  !> @param status      GRIB_SUCCESS if OK, integer value on error
+  subroutine grib_index_add_message(indexid, gribid, filename, offset, status)
+    integer(kind=kindOfInt), intent(in)             :: indexid
+    integer(kind=kindOfInt), intent(in)             :: gribid
+    character(len=*), intent(in)                    :: filename
+    integer(kind=8), intent(in)                     :: offset
+    integer(kind=kindOfInt), optional, intent(out)  :: status
+    integer(kind=kindOfInt)                         :: iret
+
+    iret = grib_f_index_add_message(indexid, gribid, filename, offset)
+    if (present(status)) then
+      status = iret
+    else
+      call grib_check(iret, 'index_add_message', '('//filename//')')
+    end if
+  end subroutine grib_index_add_message
+
   !> Get the number of distinct values of the key in argument contained in the index. The key must belong to the index.
   !>
   !>

fortran/grib_fortran.cc

@@ -1725,6 +1725,31 @@ int grib_f_index_add_file_(int* index_id, char* file, int lfile)
     }
 }
 
+/*****************************************************************************/
+int grib_f_index_add_message_(int* index_id, int* grib_id, char* file, int64_t* offset, int lfile)
+{
+    grib_index*  idx = get_index(*index_id);
+    if (!idx) return GRIB_INVALID_INDEX;
+
+    grib_handle* h   = get_handle(*grib_id);
+    if (!h)   return GRIB_INVALID_GRIB;
+
+    char buf[1024];
+
+    return codes_index_add_message(idx, h, cast_char(buf, file, lfile), (off_t)*offset);
+}
+
+/*****************************************************************************/
+int grib_f_index_new_(int* gid, char* keys, int lkeys)
+{
+    char buf[1024];
+    int err = 0;
+    grib_index* i = grib_index_new(grib_context_get_default(),
+                                   cast_char(buf, keys, lkeys), &err);
+    if (err) { *gid = -1; return err; }
+    push_index(i, gid);
+    return GRIB_SUCCESS;
+}
 /*****************************************************************************/
 int grib_f_index_read_(char* file, int* gid, int lfile)
 {
@@ -2706,7 +2731,7 @@ int grib_f_set_string_(int* gid, char* key, char* val, int len, int len2)
     size_t lsize = len2;
 
     if(!h) return GRIB_INVALID_GRIB;
-    
+
     /* For BUFR, the value may contain spaces e.g. stationOrSiteName='CAMPO NOVO' */
     /* So do not use cast_char. cast_char_no_cut does not stop at first space */
     val_str = cast_char_no_cut(buf2,val,len2);

fortran/grib_fortran_prototypes.h

@@ -128,9 +128,15 @@ int grib_f_new_from_index(int *iid, int *gid);
 int grib_f_index_new_from_file_(char *file, char *keys, int *gid, int lfile, int lkeys);
 int grib_f_index_new_from_file__(char *file, char *keys, int *gid, int lfile, int lkeys);
 int grib_f_index_new_from_file(char *file, char *keys, int *gid, int lfile, int lkeys);
+int grib_f_index_new_(int* gid, char* keys, int lkeys);
+int grib_f_index_new__(int* gid, char* keys, int lkeys);
+int grib_f_index_new(int* gid, char* keys, int lkeys);
 int grib_f_index_add_file_(int* iid, char* file, int lfile);
 int grib_f_index_add_file__(int* iid, char* file, int lfile);
 int grib_f_index_add_file(int* iid, char* file, int lfile);
+int grib_f_index_add_message_(int* index_id, int* grib_id, char* file, int64_t* offset, int lfile);
+int grib_f_index_add_message__(int* index_id, int* grib_id, char* file, int64_t* offset, int lfile);
+int grib_f_index_add_message(int* index_id, int* grib_id, char* file, int64_t* offset, int lfile);
 int grib_f_index_read_(char *file, int *gid, int lfile);
 int grib_f_index_read__(char *file, int *gid, int lfile);
 int grib_f_index_read(char *file, int *gid, int lfile);

src/eccodes/eccodes.h

@@ -236,6 +236,35 @@ codes_index* codes_index_new(codes_context* c, const char* keys, int* err);
  * @return            0 if OK, integer value on error
  */
 int codes_index_add_file(codes_index* index, const char* filename);
+
+/**
+ *  Index one message handle into the index at the given file offset.
+ *
+ *  All key extraction is performed from the in-memory handle 'h'; the file
+ *  content is NOT read by this function.  The file must already exist and be
+ *  openable so its name can be registered in the index, but it does not need
+ *  to contain the message yet.
+ *
+ *  Typical usage:
+ *    offset = ftell(fp);
+ *    codes_write_message(h, fp);          // write the message
+ *    codes_index_add_message(idx, h, path, offset); // index immediately
+ *    // ... continue writing more messages ...
+ *    // Flush before reading back through the index:
+ *    fflush(fp);
+ *    codes_handle_new_from_index(idx, &err);
+ *
+ *  The handle's product kind must match the index product kind.
+ *
+ * @param index       : index
+ * @param h           : handle of the message to index
+ * @param filename    : path of the file the message is being written to (must exist)
+ * @param offset      : byte offset of the message within the file
+ * @return            0 if OK, integer value on error
+ */
+int codes_index_add_message(codes_index* index, codes_handle* h,
+                            const char* filename, off_t offset);
+
 int codes_index_write(codes_index* index, const char* filename);
 codes_index* codes_index_read(codes_context* c, const char* filename, int* err);
 

src/eccodes/grib_api.h

@@ -287,6 +287,33 @@ int grib_index_add_file(grib_index* index, const char* filename);
 int grib_index_write(grib_index* index, const char* filename);
 grib_index* grib_index_read(grib_context* c, const char* filename, int* err);
 
+/**
+ *  Index one message handle into the index at the given file offset.
+ *
+ *  All key extraction is performed from the in-memory handle 'h'; the file
+ *  content is NOT read by this function.  The file must already exist and be
+ *  openable so its name can be registered in the index, but it does not need
+ *  to contain the message yet.
+ *
+ *  Typical usage:
+ *    offset = ftell(fp);
+ *    grib_write_message(h, fp);           // write the message
+ *    codes_index_add_message(idx, h, path, offset); // index immediately
+ *    // ... continue writing more messages ...
+ *    // Flush before reading back through the index:
+ *    fflush(fp);
+ *    grib_handle_new_from_index(idx, &err);
+ *
+ *  The handle's product kind must match the index product kind.
+ *
+ * @param index       : index
+ * @param h           : handle of the message to index
+ * @param filename    : path of the file the message is being written to (must exist)
+ * @param offset      : byte offset of the message within the file
+ * @return            0 if OK, integer value on error
+ */
+int codes_index_add_message(grib_index* index, grib_handle* h, const char* filename, off_t offset);
+
 /**
  *  Get the number of distinct values of the key in argument contained in the index. The key must belong to the index.
  *