feat(fl): per-job fl-server — flip-api scales fl-server to zero between training jobs (NVFLARE)

.env.development.example

@@ -220,6 +220,13 @@ FL_BACKEND=nvflare
 JOB_RESOURCE_SPEC_NUM_GPUS=0
 JOB_RESOURCE_SPEC_MEM_PER_GPU_IN_GIB=0
 
+# Scale fl-server-net-1 to zero between training jobs instead of running it always-on
+# (FLIP#735, NVFLARE only). Default false everywhere. Threaded into both flip-api and
+# fl-api-net-1 (compose.development.nvflare.yml) — both must agree on whether an unreachable
+# fl-server is an unplanned outage (false) or the normal idle-by-design state (true). Uncomment
+# to exercise the Phase 0 lazy-session/offline-status behavior locally.
+# PER_JOB_FL_SERVER=false
+
 # ── MFA Enforcement ───────────────────────────────────────────────────────
 # Controls whether the API requires TOTP enrollment for all authenticated
 # routes. Default is "true" (enforced) in production. In development, the

deploy/compose.development.nvflare.yml

@@ -29,6 +29,8 @@ services:
       # mem <= the client's advertised mem_per_gpu_in_GiB) so jobs train on the GPU.
       - JOB_RESOURCE_SPEC_NUM_GPUS=${JOB_RESOURCE_SPEC_NUM_GPUS:-0}
       - JOB_RESOURCE_SPEC_MEM_PER_GPU_IN_GIB=${JOB_RESOURCE_SPEC_MEM_PER_GPU_IN_GIB:-0}
+      # FLIP#735: mirrors flip-api's PER_JOB_FL_SERVER above — same source var.
+      - PER_JOB_FL_SERVER=${PER_JOB_FL_SERVER:-false}
     ports:
       - "${FL_API_PORT}:8000"
       - "5679:5679"

deploy/compose.development.yml

@@ -124,6 +124,10 @@ services:
       # Settings.ENFORCE_MFA defaults to True elsewhere. Flip to `true`
       # here (and `make restart`) to test the MFA flow end-to-end locally.
       - ENFORCE_MFA=false
+      # FLIP#735: scale fl-server-net-1 to zero between jobs. Default false (unchanged
+      # always-on behavior); set PER_JOB_FL_SERVER=true in .env.development (and mirror on
+      # fl-api-net-1 below) to exercise the Phase 0 lazy-session/offline-status behavior locally.
+      - PER_JOB_FL_SERVER=${PER_JOB_FL_SERVER:-false}
       # Settings.LOG_LEVEL defaults to INFO so unknown environments emit
       # at INFO. Local dev defaults to DEBUG when LOG_LEVEL is unset
       # (override via LOG_LEVEL in .env.development) so debug-level

deploy/compose.production.nvflare.yml

@@ -28,6 +28,9 @@ services:
       # mem <= the client's advertised mem_per_gpu_in_GiB) so jobs train on the GPU.
       - JOB_RESOURCE_SPEC_NUM_GPUS=${JOB_RESOURCE_SPEC_NUM_GPUS:-0}
       - JOB_RESOURCE_SPEC_MEM_PER_GPU_IN_GIB=${JOB_RESOURCE_SPEC_MEM_PER_GPU_IN_GIB:-0}
+      # FLIP#735: mirrors flip-api's PER_JOB_FL_SERVER (compose.production.yml) — same source
+      # var, both services must agree on whether fl-server going unreachable is an outage or idle.
+      - PER_JOB_FL_SERVER=${PER_JOB_FL_SERVER:-false}
     ports:
       - "${FL_API_PORT}:8000"
     security_opt:

deploy/compose.production.yml

@@ -28,6 +28,10 @@ services:
     environment:
       - ENV=production
       - ENFORCE_MFA=${ENFORCE_MFA:-true}
+      # FLIP#735: scale fl-server-net-1 to zero between jobs. Default false everywhere;
+      # mirrored into fl-api-net-1's own PER_JOB_FL_SERVER (compose.production.nvflare.yml) —
+      # both services must agree on whether an unreachable fl-server is an outage or idle-by-design.
+      - PER_JOB_FL_SERVER=${PER_JOB_FL_SERVER:-false}
       - LOG_LEVEL=${LOG_LEVEL:-INFO}
       # - AWS_PROFILE=${AWS_PROFILE} # in production, AWS credentials are retrieved from the instance role
       - AWS_REGION=${AWS_REGION}

deploy/providers/AWS/Makefile

@@ -118,6 +118,12 @@ export TF_VAR_MIN_CLIENTS=${MIN_CLIENTS}
 ifneq ($(JOB_RESOURCE_SPEC_NUM_GPUS),)
 export TF_VAR_JOB_RESOURCE_SPEC_NUM_GPUS=${JOB_RESOURCE_SPEC_NUM_GPUS}
 endif
+# FLIP#735: scale fl-server-net-1 to zero between jobs. Default false in variables.tf; set
+# PER_JOB_FL_SERVER=true in the env file (stag first) once Phase 2 has landed. Conditional export
+# for the same reason as the GPU resource-spec vars above (empty string ≠ bool for Terraform).
+ifneq ($(PER_JOB_FL_SERVER),)
+export TF_VAR_PER_JOB_FL_SERVER=${PER_JOB_FL_SERVER}
+endif
 ifneq ($(JOB_RESOURCE_SPEC_MEM_PER_GPU_IN_GIB),)
 export TF_VAR_JOB_RESOURCE_SPEC_MEM_PER_GPU_IN_GIB=${JOB_RESOURCE_SPEC_MEM_PER_GPU_IN_GIB}
 endif

deploy/providers/AWS/locals.tf

@@ -126,6 +126,7 @@ locals {
       # complete within 30 minutes or they time out at the edge.
       MAX_MODEL_FILE_BYTES = "5000000000"
       FL_KIT_SLOT_NAMES    = var.FL_KIT_SLOT_NAMES
+      PER_JOB_FL_SERVER    = tostring(var.PER_JOB_FL_SERVER)
     })
     fl_server = {
       LOCAL_DEV                      = "false"
@@ -160,6 +161,10 @@ locals {
       # CPU-only. Default 0; set via TF_VAR_JOB_RESOURCE_SPEC_* for GPU jobs.
       JOB_RESOURCE_SPEC_NUM_GPUS           = tostring(var.JOB_RESOURCE_SPEC_NUM_GPUS)
       JOB_RESOURCE_SPEC_MEM_PER_GPU_IN_GIB = tostring(var.JOB_RESOURCE_SPEC_MEM_PER_GPU_IN_GIB)
+      # Mirrors flip_api's PER_JOB_FL_SERVER above — same Terraform variable, single source of
+      # truth (FLIP#735). Both services must agree on whether an unreachable fl-server is an
+      # unplanned outage or the normal idle state between jobs.
+      PER_JOB_FL_SERVER = tostring(var.PER_JOB_FL_SERVER)
     }
   }
 }

deploy/providers/AWS/variables.tf

@@ -196,6 +196,18 @@ variable "JOB_RESOURCE_SPEC_MEM_PER_GPU_IN_GIB" {
   default     = 0
 }
 
+# FLIP#735: scale fl-server-net-1 to zero between training jobs instead of running it 24/7.
+# Threaded as an identical env var into both flip-api and fl-api-net-1 (see ecs_task_env in
+# locals.tf) — both services must agree on whether an unreachable fl-server is an unplanned
+# outage (false, default: fl-api-net-1 crash-loops as today) or the normal idle-by-design state
+# between jobs (true: flip-api drives ecs:UpdateService, fl-api-net-1 tolerates the server being
+# down). Default false everywhere; set true in `.env.stag` only once Phase 2 has landed.
+variable "PER_JOB_FL_SERVER" {
+  description = "Scale fl-server-net-1 to zero between jobs instead of running it always-on"
+  type        = bool
+  default     = false
+}
+
 # Gates the whole FL-on-ECS stack, not just the file system: both FL task
 # definitions (fl-api-net-1, fl-server-net-1 in ecs_tasks.tf) carry
 # `count = var.enable_efs ? 1 : 0`, so their EFS volumes (including the shared

fl-services/nvflare/fl-api-base/fl_api/config.py

@@ -25,6 +25,12 @@ class Settings(BaseSettings):
 
     TIMEOUT_SESSION_CONNECT: float = 20.0
 
+    # Mirrors flip-api's PER_JOB_FL_SERVER (FLIP#735) — threaded in as an env var from the same
+    # shared Terraform variable so both services agree on whether fl-server going unreachable is
+    # an unplanned outage (flag off: fail fast at boot, as today) or the normal idle-by-design
+    # state between jobs (flag on: tolerate it and connect lazily on first use).
+    PER_JOB_FL_SERVER: bool = False
+
     # GPU resources that the submitted NVFLARE jobs need in order to schedule correctly.
     # TODO Currently this is set globally for all jobs, but we should allow per-job overrides in the future.
     # See https://github.com/londonaicentre/flip/issues/41

fl-services/nvflare/fl-api-base/fl_api/routers/system.py

@@ -13,7 +13,10 @@
 # System and service status functions
 
 from fastapi import APIRouter, Depends, Query, status
+from nvflare.apis.fl_exception import FLCommunicationError
+from nvflare.fuel.flare_api.api_spec import NoConnection
 
+from fl_api.config import get_settings
 from fl_api.core.dependencies import get_session
 from fl_api.utils.flip_session import FLIP_Session
 from fl_api.utils.schemas import ClientInfoModel, ServerInfoModel, SystemInfoModel, TargetType
@@ -26,13 +29,23 @@ def check_server_status(session: FLIP_Session = Depends(get_session)) -> ServerI
     """
     Checks the status of the server.
 
+    With PER_JOB_FL_SERVER on, an unreachable fl-server is the normal idle-between-jobs state
+    (FLIP#735), not an error — this returns status="STOPPED" so flip-api can poll this endpoint as
+    a readiness probe. With the flag off, an unreachable server is still an unplanned outage and
+    this raises, unchanged.
+
     Args:
         session (FLIP_Session): the FLIP session instance.
 
     Returns:
         ServerInfoModel: status information about the server.
     """
-    return session.check_server_status()
+    try:
+        return session.check_server_status()
+    except (NoConnection, FLCommunicationError):
+        if not get_settings().PER_JOB_FL_SERVER:
+            raise
+        return ServerInfoModel(status="STOPPED", start_time=0.0)
 
 
 @router.get("/check_client_status", response_model=list[ClientInfoModel])

fl-services/nvflare/fl-api-base/fl_api/startup/session_manager.py

@@ -11,6 +11,9 @@
 #
 
 
+from nvflare.apis.fl_exception import FLCommunicationError
+from nvflare.fuel.flare_api.api_spec import NoConnection
+
 from fl_api.config import get_settings
 from fl_api.utils.flip_session import FLIP_Session
 from fl_api.utils.logger import logger
@@ -20,6 +23,13 @@ def create_fl_session() -> FLIP_Session:
     """
     Initialize NVIDIA FLARE admin workspace and return a session object for interacting with it.
 
+    With PER_JOB_FL_SERVER off, an unreachable fl-server at boot is fatal (unchanged) — this is the
+    normal alerting signal for an unplanned outage. With it on, fl-server is expected to be down
+    most of the time (scaled to zero between jobs, FLIP#735), so a connection failure here is
+    logged and tolerated: the session connects lazily on first use (see FLIP_Session._do_command).
+    A non-connectivity failure (bad config, auth) still raises either way — only "can't reach the
+    server" is tolerated.
+
     Returns:
         FLIP_Session: An initialized FLIP_Session object.
     """
@@ -43,10 +53,18 @@ def create_fl_session() -> FLIP_Session:
         debug=debug,
     )
 
-    # Try connecting the session here, so that we can catch any connection issues at startup
-    session.try_connect(get_settings().TIMEOUT_SESSION_CONNECT)
-
     logger.info(f"Upload directory set to: {session.upload_dir}")
     logger.info(f"Download directory set to: {session.download_dir}")
 
+    # Try connecting the session here, so that we can catch any connection issues at startup
+    try:
+        session.try_connect(get_settings().TIMEOUT_SESSION_CONNECT)
+    except (NoConnection, FLCommunicationError):
+        if not get_settings().PER_JOB_FL_SERVER:
+            raise
+        logger.warning(
+            "fl-server unreachable at boot; PER_JOB_FL_SERVER is on, treating this as the normal "
+            "idle-between-jobs state. Will connect lazily on first use."
+        )
+
     return session

fl-services/nvflare/fl-api-base/fl_api/utils/flip_session.py

@@ -31,24 +31,40 @@ def __init__(
         self._debug = debug
         super(FLIP_Session, self).__init__(username, startup_path, secure_mode, debug)
         self._error_buffer = None
+        # Tracked ourselves rather than inspecting nvflare's private `api.closed` — a freshly
+        # constructed AdminAPI is neither "closed" nor actually connected, so `api.closed` can't
+        # distinguish "never connected" (PER_JOB_FL_SERVER's lazy-boot case) from a live session.
+        self._connected = False
+
+    def try_connect(self, timeout: float) -> None:
+        """Connect the underlying admin API, tracking success so `_do_command` knows whether a
+        lazy first-use connect is needed (see PER_JOB_FL_SERVER in session_manager.py)."""
+        super().try_connect(timeout)
+        self._connected = True
 
     def _reconnect(self) -> None:
         """Re-initialise the underlying admin API and log in again after the session was closed."""
+        self._connected = False
         Session.__init__(self, self.username, self._startup_path, self._secure_mode, self._debug)
         self.try_connect(timeout=5.0)
 
     def _do_command(self, cmd: str):
         """
         Override the _do_command method to add error handling for session inactivity or closure.
 
-        On ``InternalError`` with "session_inactive", reconnects via ``try_connect`` and retries once.
-        On ``SessionClosed`` (e.g. idle timeout, fl-server restart, network blip), fully re-initialises
-        the admin session via ``_reconnect`` and retries once. Any exception on the retry is logged and
-        re-raised immediately — there is no further retry loop.
+        If never successfully connected (PER_JOB_FL_SERVER let boot proceed with an unreachable
+        fl-server), connects lazily here on first use. On ``InternalError`` with "session_inactive",
+        reconnects via ``try_connect`` and retries once. On ``SessionClosed`` (e.g. idle timeout,
+        fl-server restart, network blip), fully re-initialises the admin session via ``_reconnect``
+        and retries once. Any exception on the retry is logged and re-raised immediately — there is
+        no further retry loop.
 
         Args:
             cmd (str): The command to be executed.
         """
+        if not self._connected:
+            logger.info("Session not yet connected; connecting now before running command: %s", cmd)
+            self.try_connect(timeout=5.0)
         try:
             return super()._do_command(cmd)
         except InternalError as e:
@@ -66,6 +82,11 @@ def _do_command(self, cmd: str):
                 logger.error("Retry after reconnect failed for command: %s", cmd)
                 raise
 
+    @property
+    def is_connected(self) -> bool:
+        """Whether the admin session has ever successfully connected (see PER_JOB_FL_SERVER)."""
+        return self._connected
+
     def check_server_status(self) -> ServerInfoModel:
         """
         Checks the status of the server.

fl-services/nvflare/fl-api-base/tests/routers/test_system.py

@@ -10,10 +10,11 @@
 # limitations under the License.
 #
 
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
 import pytest
 from fastapi import status
+from nvflare.fuel.flare_api.api_spec import NoConnection
 
 from fl_api.app import app
 from fl_api.core.dependencies import get_session
@@ -64,6 +65,33 @@ def test_check_server_status(client, mock_session):
     assert resp_json["start_time"] == 123.0
 
 
+def test_check_server_status_returns_stopped_when_unreachable_and_flag_on(client, mock_session):
+    """✅ PER_JOB_FL_SERVER on: an unreachable server reports STOPPED, not a 500 (FLIP#735)."""
+    app.dependency_overrides[system.get_session] = lambda: mock_session
+    mock_session.check_server_status.side_effect = NoConnection("cannot connect to server")
+
+    with patch("fl_api.routers.system.get_settings") as mock_get_settings:
+        mock_get_settings.return_value.PER_JOB_FL_SERVER = True
+        resp = client.get("/check_server_status")
+
+    assert resp.status_code == status.HTTP_200_OK
+    resp_json = resp.json()
+    assert resp_json["status"] == "STOPPED"
+    assert resp_json["start_time"] == 0.0
+
+
+def test_check_server_status_raises_when_unreachable_and_flag_off(client, mock_session):
+    """❌ PER_JOB_FL_SERVER off (default): an unreachable server still errors, unchanged."""
+    app.dependency_overrides[system.get_session] = lambda: mock_session
+    mock_session.check_server_status.side_effect = NoConnection("cannot connect to server")
+
+    with patch("fl_api.routers.system.get_settings") as mock_get_settings:
+        mock_get_settings.return_value.PER_JOB_FL_SERVER = False
+        resp = client.get("/check_server_status")
+
+    assert resp.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR
+
+
 # ---------------------------------------------------------------------
 # check_client_status
 # ---------------------------------------------------------------------

fl-services/nvflare/fl-api-base/tests/startup/test_session_manager.py

@@ -14,6 +14,8 @@
 from unittest.mock import MagicMock, patch
 
 import pytest
+from nvflare.apis.fl_exception import FLCommunicationError
+from nvflare.fuel.flare_api.api_spec import AuthenticationError, NoConnection
 
 from fl_api.startup.session_manager import create_fl_session
 
@@ -30,6 +32,7 @@ class Settings:
         JOB_RESOURCE_SPEC_NUM_GPUS = 2
         JOB_RESOURCE_SPEC_MEM_PER_GPU_IN_GIB = 8
         TIMEOUT_SESSION_CONNECT = 5.0
+        PER_JOB_FL_SERVER = False
 
     os.makedirs(Settings.FL_ADMIN_DIRECTORY, exist_ok=True)
     return Settings()
@@ -50,3 +53,64 @@ def test_create_fl_session_success(fake_settings):
     assert session == mock_session
     assert session.upload_dir == "/tmp/upload"
     assert session.download_dir == "/tmp/download"
+
+
+def test_create_fl_session_raises_when_unreachable_and_flag_off(fake_settings):
+    """❌ PER_JOB_FL_SERVER off (default): an unreachable server at boot still raises, unchanged."""
+    fake_settings.PER_JOB_FL_SERVER = False
+    mock_session = MagicMock()
+    mock_session.try_connect.side_effect = NoConnection("cannot connect to server")
+
+    with (
+        patch("fl_api.startup.session_manager.get_settings", return_value=fake_settings),
+        patch("fl_api.startup.session_manager.FLIP_Session", return_value=mock_session),
+    ):
+        with pytest.raises(NoConnection):
+            create_fl_session()
+
+
+def test_create_fl_session_tolerates_unreachable_server_when_flag_on(fake_settings):
+    """✅ PER_JOB_FL_SERVER on: an unreachable server at boot is tolerated (idle-by-design, FLIP#735)."""
+    fake_settings.PER_JOB_FL_SERVER = True
+    mock_session = MagicMock()
+    mock_session.upload_dir = "/tmp/upload"
+    mock_session.download_dir = "/tmp/download"
+    mock_session.try_connect.side_effect = NoConnection("cannot connect to server")
+
+    with (
+        patch("fl_api.startup.session_manager.get_settings", return_value=fake_settings),
+        patch("fl_api.startup.session_manager.FLIP_Session", return_value=mock_session),
+    ):
+        session = create_fl_session()
+
+    assert session == mock_session
+
+
+def test_create_fl_session_tolerates_fl_communication_error_when_flag_on(fake_settings):
+    """✅ PER_JOB_FL_SERVER on: a raw FLCommunicationError (not remapped to NoConnection) is tolerated too."""
+    fake_settings.PER_JOB_FL_SERVER = True
+    mock_session = MagicMock()
+    mock_session.try_connect.side_effect = FLCommunicationError("comm error")
+
+    with (
+        patch("fl_api.startup.session_manager.get_settings", return_value=fake_settings),
+        patch("fl_api.startup.session_manager.FLIP_Session", return_value=mock_session),
+    ):
+        session = create_fl_session()
+
+    assert session == mock_session
+
+
+def test_create_fl_session_still_raises_on_non_connectivity_error_when_flag_on(fake_settings):
+    """❌ PER_JOB_FL_SERVER on: a non-connectivity failure (e.g. auth) still raises — only
+    unreachability is tolerated, not arbitrary startup failures."""
+    fake_settings.PER_JOB_FL_SERVER = True
+    mock_session = MagicMock()
+    mock_session.try_connect.side_effect = AuthenticationError("bad cert")
+
+    with (
+        patch("fl_api.startup.session_manager.get_settings", return_value=fake_settings),
+        patch("fl_api.startup.session_manager.FLIP_Session", return_value=mock_session),
+    ):
+        with pytest.raises(AuthenticationError):
+            create_fl_session()