es_out: support Upstream Servers with configuration overriding

.github/workflows/call-build-images.yaml

@@ -413,6 +413,26 @@ jobs:
           username: ${{ inputs.username }}
           password: ${{ secrets.token }}
 
+      # https://github.com/actions/runner-images/issues/12199
+      # https://github.com/moby/moby/issues/48093
+      # https://support.microsoft.com/en-us/topic/gettemppath-changes-in-windows-february-cumulative-update-preview-4cc631fb-9d97-4118-ab6d-f643cd0a7259#ID0EDF
+      - name: Change system temp directory to the drive having sufficient disk space
+        id: relocate-system-temp-dir
+        run: |
+          $ErrorActionPreference = "Stop"
+          if (Test-Path -Path "D:\") {
+            $newSystemTempDir = "D:\SystemTemp"
+            New-Item -ItemType Directory -Path "${newSystemTempDir}" -Force
+            $acl = New-Object System.Security.AccessControl.DirectorySecurity
+            $acl.SetSecurityDescriptorSddlForm("O:SYG:SYD:PAI(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)")
+            Set-Acl "${newSystemTempDir}" -AclObject ${acl}
+            [Environment]::SetEnvironmentVariable("SYSTEMTEMP", "${newSystemTempDir}", [EnvironmentVariableTarget]::Machine)
+            Restart-Service docker
+          } else {
+            Write-Warning "Drive D doesn't exist, so there is no disk to relocate system temp directory"
+          }
+        shell: pwsh
+
       - name: Pull the last release image to speed up the build with a cache
         continue-on-error: true
         run: |

.github/workflows/pr-image-tests.yaml

@@ -108,6 +108,26 @@ jobs:
             flavor: |
               suffix=-windows-${{ matrix.windows-base-version }}
 
+        # https://github.com/actions/runner-images/issues/12199
+        # https://github.com/moby/moby/issues/48093
+        # https://support.microsoft.com/en-us/topic/gettemppath-changes-in-windows-february-cumulative-update-preview-4cc631fb-9d97-4118-ab6d-f643cd0a7259#ID0EDF
+        - name: Change system temp directory to the drive having sufficient disk space
+          id: relocate-system-temp-dir
+          run: |
+            $ErrorActionPreference = "Stop"
+            if (Test-Path -Path "D:\") {
+              $newSystemTempDir = "D:\SystemTemp"
+              New-Item -ItemType Directory -Path "${newSystemTempDir}" -Force
+              $acl = New-Object System.Security.AccessControl.DirectorySecurity
+              $acl.SetSecurityDescriptorSddlForm("O:SYG:SYD:PAI(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)")
+              Set-Acl "${newSystemTempDir}" -AclObject ${acl}
+              [Environment]::SetEnvironmentVariable("SYSTEMTEMP", "${newSystemTempDir}", [EnvironmentVariableTarget]::Machine)
+              Restart-Service docker
+            } else {
+              Write-Warning "Drive D doesn't exist, so there is no disk to relocate system temp directory"
+            }
+          shell: pwsh
+
         - name: Build the windows images
           id: build
           run: |

dockerfiles/Dockerfile.windows

@@ -47,10 +47,7 @@ RUN $msvs_build_tools_dist_name=\"vs_buildtools.exe\"; `
       \"--channelUri ${msvs_build_tools_channel}\", `
       \"--installChannelUri ${msvs_build_tools_channel}\", `
       '--add Microsoft.VisualStudio.Workload.VCTools', `
-      '--add Microsoft.VisualStudio.Workload.MSBuildTools', `
       '--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64', `
-      '--add Microsoft.VisualStudio.Component.VC.CoreBuildTools', `
-      '--add Microsoft.VisualStudio.Component.VC.MSVC.143', `
       '--add Microsoft.VisualStudio.Component.Windows10SDK.19041' `
       -NoNewWindow -Wait -PassThru; `
     if (${p}.ExitCode -ne 0 -and ${p}.ExitCode -ne 3010) { `
@@ -94,6 +91,26 @@ RUN if ([System.Version] \"${env:CMAKE_VERSION}\" -ge [System.Version] \"3.20.0\
     Write-Host \"${env:PATH}\"; `
     [Environment]::SetEnvironmentVariable(\"PATH\", \"${env:PATH}\", [EnvironmentVariableTarget]::Machine);
 
+ENV NINJA_HOME="C:\ninja"
+ARG NINJA_VERSION="1.13.2"
+ARG NINJA_URL="https://github.com/ninja-build/ninja/releases/download"
+
+RUN $ninja_dist_name=\"ninja-win.zip\"; `
+    $ninja_dist=\"${env:TMP}\${ninja_dist_name}\"; `
+    $ninja_download_url=\"${env:NINJA_URL}/v${env:NINJA_VERSION}/${ninja_dist_name}\"; `
+    Write-Host \"Downloading Ninja...\"; `
+    Write-Host \"${ninja_download_url} -> ${ninja_dist}\"; `
+    Invoke-WebRequest -OutFile \"${ninja_dist}\" \"${ninja_download_url}\"; `
+    New-Item -Path \"${env:NINJA_HOME}\" -ItemType \"directory\"; `
+    Write-Host \"Extracting Ninja...\"; `
+    Write-Host \"${ninja_dist} -> ${env:NINJA_HOME}\"; `
+    Expand-Archive \"${ninja_dist}\" -Destination \"${env:NINJA_HOME}\"; `
+    Remove-Item -Force \"${ninja_dist}\"; `
+    $env:PATH=\"${env:PATH};${env:NINJA_HOME}\"; `
+    Write-Host \"Setting PATH...\"; `
+    Write-Host \"${env:PATH}\"; `
+    [Environment]::SetEnvironmentVariable(\"PATH\", \"${env:PATH}\", [EnvironmentVariableTarget]::Machine);
+
 ENV WIN_FLEX_BISON_VERSION="2.5.22" `
     WIN_FLEX_BISON_HOME="C:\WinFlexBison" `
     WIN_FLEX_BISON_DOWNLOAD_URL="https://github.com/lexxmark/winflexbison/releases/download"
@@ -181,7 +198,7 @@ COPY . /src/
 ARG BUILD_PARALLEL=1
 SHELL ["cmd", "/S", "/C"]
 RUN call "%MSVS_HOME%\VC\Auxiliary\Build\vcvars64.bat" && `
-    cmake -G "NMake Makefiles" `
+    cmake -G "Ninja" `
       -DOPENSSL_ROOT_DIR='C:\dev\vcpkg\packages\openssl_x64-windows-static' `
       -DFLB_LIBYAML_DIR='C:\dev\vcpkg\packages\libyaml_x64-windows-static' `
       -DFLB_SIMD=On `
@@ -191,7 +208,7 @@ RUN call "%MSVS_HOME%\VC\Auxiliary\Build\vcvars64.bat" && `
       -DFLB_DEBUG=Off `
       -DFLB_RELEASE=On `
       ..\ && `
-    cmake --build . --config Release -j "%BUILD_PARALLEL%"
+    cmake --build . -j "%BUILD_PARALLEL%"
 
 SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
 

include/fluent-bit/flb_lib.h

@@ -23,6 +23,8 @@
 #include <fluent-bit/flb_macros.h>
 #include <fluent-bit/flb_config.h>
 
+struct flb_input_instance;
+
 /* Lib engine status */
 #define FLB_LIB_ERROR     -1
 #define FLB_LIB_NONE       0
@@ -69,7 +71,14 @@ FLB_EXPORT int flb_output_set_test(flb_ctx_t *ctx, int ffd, char *test_name,
                                    void (*out_callback) (void *, int, int,
                                                          void *, size_t, void *),
                                    void *out_callback_data,
-                                   void *test_ctx);
+                                   void *flush_ctx);
+FLB_EXPORT int flb_output_set_test_flush_ctx_callback(flb_ctx_t *ctx, int ffd,
+                                                      char *test_name,
+                                                      void *(*flush_ctx_callback)(
+                                                              struct flb_config *,
+                                                              struct flb_input_instance *,
+                                                              void *, void *),
+                                                      void *flush_ctx);
 FLB_EXPORT int flb_output_set_callback(flb_ctx_t *ctx, int ffd, char *name,
                                        void (*cb)(char *, void *, void *));
 FLB_EXPORT int flb_output_set_http_test(flb_ctx_t *ctx, int ffd, char *test_name,

include/fluent-bit/flb_output.h

@@ -154,9 +154,24 @@ struct flb_test_out_formatter {
      */
     void *rt_data;
 
-    /* optional context for flush callback */
+    /* optional context for "flush context callback" */
     void *flush_ctx;
 
+    /*
+     * Callback
+     * =========
+     * Optional "flush context callback": it references the function that extracts
+     * optional flush context for "formatter callback".
+     */
+    void *(*flush_ctx_callback) (/* Fluent Bit context */
+                                 struct flb_config *,
+                                 /* plugin that ingested the records */
+                                 struct flb_input_instance *,
+                                 /* plugin instance context */
+                                 void *plugin_context,
+                                 /* context for "flush context callback" */
+                                 void *flush_ctx);
+
     /*
      * Callback
      * =========

plugins/out_es/CMakeLists.txt

@@ -1,5 +1,7 @@
 set(src
   es_bulk.c
+  es_type.c
+  es_conf_parse.c
   es_conf.c
   es.c
   murmur3.c)