diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml index 56a2e87..781a6be 100644 --- a/.github/workflows/ci-build.yaml +++ b/.github/workflows/ci-build.yaml @@ -44,18 +44,18 @@ jobs: #Build docker images - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4.2.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.2.0 + uses: docker/setup-buildx-action@v4.2.0 - name: Login to GitHub Packages Docker Registry - uses: docker/login-action@v3.0.0 + uses: docker/login-action@v4.4.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker meta - frontend id: docker_meta - uses: docker/metadata-action@v5.5.1 + uses: docker/metadata-action@v6.2.0 with: images: ghcr.io/splunk/sc4snmp-ui/frontend/container tags: | @@ -68,7 +68,7 @@ jobs: type=ref,event=branch - name: Build and push action - frontend id: docker_action_build_frontend - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v7.3.0 with: context: frontend push: false @@ -94,18 +94,18 @@ jobs: #Build docker images - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4.2.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.2.0 + uses: docker/setup-buildx-action@v4.2.0 - name: Login to GitHub Packages Docker Registry - uses: docker/login-action@v3.0.0 + uses: docker/login-action@v4.4.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker meta - backend id: docker_meta - uses: docker/metadata-action@v5.5.1 + uses: docker/metadata-action@v6.2.0 with: images: ghcr.io/splunk/sc4snmp-ui/backend/container tags: | @@ -118,7 +118,7 @@ jobs: type=ref,event=branch - name: Build and push action - backend id: docker_action_build_backend - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v7.3.0 with: context: backend push: false diff --git a/.github/workflows/ci-release.yaml b/.github/workflows/ci-release.yaml index 83ccd06..409ed48 100644 --- a/.github/workflows/ci-release.yaml +++ b/.github/workflows/ci-release.yaml @@ -39,18 +39,18 @@ jobs: #Build docker images - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4.2.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.2.0 + uses: docker/setup-buildx-action@v4.2.0 - name: Login to GitHub Packages Docker Registry - uses: docker/login-action@v3.0.0 + uses: docker/login-action@v4.4.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker meta - frontend id: docker_meta - uses: docker/metadata-action@v5.5.1 + uses: docker/metadata-action@v6.2.0 with: images: ghcr.io/splunk/sc4snmp-ui/frontend/container tags: | @@ -64,7 +64,7 @@ jobs: type=ref,event=pr - name: Build and push action - frontend id: docker_action_build_frontend - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v7.3.0 with: context: frontend push: true @@ -90,18 +90,18 @@ jobs: #Build docker images - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4.2.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.2.0 + uses: docker/setup-buildx-action@v4.2.0 - name: Login to GitHub Packages Docker Registry - uses: docker/login-action@v3.0.0 + uses: docker/login-action@v4.4.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker meta - backend id: docker_meta - uses: docker/metadata-action@v5.5.1 + uses: docker/metadata-action@v6.2.0 with: images: ghcr.io/splunk/sc4snmp-ui/backend/container tags: | @@ -115,7 +115,7 @@ jobs: type=ref,event=pr - name: Build and push action - backend id: docker_action_build_backend - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v7.3.0 with: context: backend push: true diff --git a/CHANGELOG.md b/CHANGELOG.md index b3d5072..279e580 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ # Changelog ### Changed +- fix button misalignment on multilines in table +- fix schedules not resuming for unchanged inventory records after Redis/RedBeat is reset while MongoDB data is preserved +- fix font flash on page switch by inlining Proxima Nova as WOFF2 instead of loading OTF files on demand +- add "Restore configuration" button to recover Profiles, Groups, and Inventory from section files on disk when MongoDB data is lost +- hide Logout button when authentication is disabled +- fix port being required when adding a Group inventory record +- fix Groups tab freezing for large device counts by paginating the group list and batching inventory-membership lookups +- add "Bulk add devices" to a group, supporting a manual grid or a pasted address list with shared SNMP config ## [1.2.1] diff --git a/README.md b/README.md index 603f32e..bba6953 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ pip3 install -r requirements.txt Run mongoDB in docker: ```shell -docker run --rm -d -p 27017:27017 --name example-mongo mongo:4.4.6 +docker run --rm -d -p 27017:27017 --name example-mongo mongo:8.3.4 ``` To start backend service run: diff --git a/backend/Dockerfile b/backend/Dockerfile index 558f3cf..6187f37 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim +FROM python:3.14-slim WORKDIR /app COPY --chown=10000:10000 requirements.txt app.py ./ diff --git a/backend/SC4SNMP_UI_backend/__init__.py b/backend/SC4SNMP_UI_backend/__init__.py index dd441c1..703c238 100644 --- a/backend/SC4SNMP_UI_backend/__init__.py +++ b/backend/SC4SNMP_UI_backend/__init__.py @@ -16,7 +16,7 @@ load_dotenv() -__version__ = "1.2.2" +__version__ = "1.2.3-beta.8" MONGO_URI = os.getenv("MONGO_URI") log = logging.getLogger('gunicorn.error') @@ -88,7 +88,7 @@ class AuthNotConfiguredException(Exception): pass -limiter = Limiter(key_func=get_remote_address, default_limits=[]) +limiter = Limiter(key_func=get_remote_address, default_limits=[], storage_uri=REDBEAT_URL) def create_app(): @@ -135,7 +135,6 @@ def create_app(): CORS(app, origins=cors_origins, supports_credentials=True) limiter.init_app(app) - limiter.storage_uri = REDBEAT_URL if REDIS_MODE == "replication": broker_transport_options = { diff --git a/backend/SC4SNMP_UI_backend/apply_changes/handling_chain.py b/backend/SC4SNMP_UI_backend/apply_changes/handling_chain.py index 297b15b..582ef4e 100644 --- a/backend/SC4SNMP_UI_backend/apply_changes/handling_chain.py +++ b/backend/SC4SNMP_UI_backend/apply_changes/handling_chain.py @@ -202,13 +202,48 @@ def handle(self, request: dict): ScheduleHandler schedules the kubernetes job with updated sc4snmp configuration """ record = list(mongo_config_collection.find())[0] - if not record["currently_scheduled"] and request["schedule_new_job"]: + currently_scheduled = record["currently_scheduled"] + if currently_scheduled and not self._is_task_still_scheduled(record.get("task_id")): + current_app.logger.info( + f"ScheduleHandler: currently_scheduled was True but task_id " + f"{record.get('task_id')} is no longer known to any worker; resetting stale state." + ) + mongo_config_collection.update_one({"_id": record["_id"]}, + {"$set": {"currently_scheduled": False, "task_id": None}}) + currently_scheduled = False + + if not currently_scheduled and request["schedule_new_job"]: # If the task isn't currently scheduled, schedule it and update its state in mongo. async_result = run_job.apply_async(countdown=request["job_delay"], queue='apply_changes') mongo_config_collection.update_one({"_id": record["_id"]}, {"$set": {"currently_scheduled": True, "task_id": async_result.id}}) current_app.logger.info( f"ScheduleHandler: scheduling new task with the delay of {request['job_delay']} seconds.") + currently_scheduled = True else: current_app.logger.info("ScheduleHandler: new job wasn't scheduled.") - return request["job_delay"], record["currently_scheduled"] \ No newline at end of file + return request["job_delay"], currently_scheduled + + @staticmethod + def _is_task_still_scheduled(task_id): + """ + Check whether task_id is still known to any live Celery worker as a scheduled (ETA) + task. Returns False (treat as stale) if there's no task_id, or if no worker reports + it - either because it genuinely doesn't exist anymore, or because inspection itself + failed (e.g. broker unreachable), which is the same "can't confirm it's alive" case. + """ + if not task_id: + return False + try: + celery_app = current_app.extensions["celery"] + scheduled = celery_app.control.inspect().scheduled() or {} + except Exception as e: + current_app.logger.warning( + f"ScheduleHandler: failed to inspect scheduled tasks, treating task_id " + f"{task_id} as stale: {e}") + return False + return any( + entry.get("request", {}).get("id") == task_id + for worker_tasks in scheduled.values() + for entry in worker_tasks + ) \ No newline at end of file diff --git a/backend/SC4SNMP_UI_backend/apply_changes/routes.py b/backend/SC4SNMP_UI_backend/apply_changes/routes.py index e8773c4..920f610 100644 --- a/backend/SC4SNMP_UI_backend/apply_changes/routes.py +++ b/backend/SC4SNMP_UI_backend/apply_changes/routes.py @@ -1,13 +1,107 @@ from flask import Blueprint, jsonify, current_app from SC4SNMP_UI_backend.auth.utils import login_required from SC4SNMP_UI_backend.apply_changes.apply_changes import ApplyChanges -from SC4SNMP_UI_backend.apply_changes.handling_chain import EmptyValuesFileException, YamlParserException +from SC4SNMP_UI_backend.apply_changes import handling_chain +from SC4SNMP_UI_backend.apply_changes.handling_chain import ( + EmptyValuesFileException, + YamlParserException, + mongo_groups, + mongo_inventory, + mongo_profiles, +) +from SC4SNMP_UI_backend.common.file_to_config_utils import ( + groups_yaml_to_documents, + profiles_yaml_to_documents, + inventory_csv_to_documents, +) +from SC4SNMP_UI_backend.common.mongo_utils import run_write import os import traceback +import yaml apply_changes_blueprint = Blueprint('common_blueprint', __name__) JOB_CREATION_RETRIES = int(os.getenv("JOB_CREATION_RETRIES", 10)) +# Maps a section name to its key in values.yaml, used to derive the section +# file name the same way SaveConfigToFileHandler does (TMP_FILE_PREFIX + +# key.replace(".", "_") + ".yaml"). +SECTION_FILE_KEYS = { + "groups": "scheduler.groups", + "profiles": "scheduler.profiles", + "inventory": "poller.inventory", +} + + +def _section_file_path(section_key): + file_name = handling_chain.TMP_FILE_PREFIX + SECTION_FILE_KEYS[section_key].replace(".", "_") + ".yaml" + return os.path.join(handling_chain.VALUES_DIRECTORY, file_name) + + +def _load_yaml_section(section_key): + """ + Reads and parses a section file with plain PyYAML (not ruamel), since the + result is inserted straight into Mongo and must not carry ruamel's + CommentedMap/scalar-string wrappers. Returns None if the file isn't + present, so the caller can skip restoring that section. + """ + file_path = _section_file_path(section_key) + if not os.path.exists(file_path): + return None + with open(file_path, "r") as file: + return yaml.safe_load(file) + + +def _reconcile_inventory(inventory_documents, session=None): + """ + Orphan-safe replace of inventory_ui: hosts/groups no longer present in the + restored section file are marked delete=True (mirroring the soft-delete + inventory/routes.py itself performs) so the connector loader tears down + their RedBeat walk schedules on the next Apply Changes, rather than the + rows just vanishing from Mongo with their schedules left orphaned in + Redis. Rows still present in the file are upserted by {address, port}, + clearing any prior delete=True on rows that reappear. + """ + parsed_keys = {(doc["address"], doc["port"]) for doc in inventory_documents} + existing_records = list(mongo_inventory.find({"delete": False}, session=session)) + for record in existing_records: + if (record["address"], record["port"]) not in parsed_keys: + mongo_inventory.update_one( + {"_id": record["_id"]}, {"$set": {"delete": True}}, session=session + ) + + for doc in inventory_documents: + mongo_inventory.update_one( + {"address": doc["address"], "port": doc["port"]}, + {"$set": doc}, + upsert=True, + session=session, + ) + + +def _restore_documents(session, groups_yaml, profiles_yaml, inventory_yaml): + """ + Performs all Mongo writes for a restore, forwarding session (possibly + None) to every write so common.mongo_utils.run_write can commit them + atomically when the deployment supports transactions. + """ + if groups_yaml is not None: + group_documents = groups_yaml_to_documents(groups_yaml) + mongo_groups.delete_many({}, session=session) + if group_documents: + mongo_groups.insert_many(group_documents, session=session) + + if profiles_yaml is not None: + profile_documents = profiles_yaml_to_documents(profiles_yaml) + mongo_profiles.delete_many({}, session=session) + if profile_documents: + mongo_profiles.insert_many(profile_documents, session=session) + + if inventory_yaml is not None: + inventory_csv = (inventory_yaml or {}).get("inventory", "") + inventory_documents = inventory_csv_to_documents(inventory_csv) + _reconcile_inventory(inventory_documents, session=session) + + @apply_changes_blueprint.route("/apply-changes", methods=['POST']) @login_required def apply_changes(): @@ -23,6 +117,43 @@ def apply_changes(): result = jsonify({"message": message}) return result, 200 + +@apply_changes_blueprint.route("/load-config", methods=['POST']) +@login_required +def load_config(): + """ + Restores profiles, groups, and inventory from the on-disk section files + (sc4snmp_ui_scheduler_groups.yaml / sc4snmp_ui_scheduler_profiles.yaml / + sc4snmp_ui_poller_inventory.yaml) into their *_ui Mongo collections, then + triggers the same Apply Changes flow as /apply-changes so the connector's + inventory Job picks the restored configuration up (both UI visibility and + polling). This is the only inverse (file -> Mongo) path in the backend; + everywhere else Mongo is the sole source of truth. + """ + groups_yaml = _load_yaml_section("groups") + profiles_yaml = _load_yaml_section("profiles") + inventory_yaml = _load_yaml_section("inventory") + + if groups_yaml is None and profiles_yaml is None and inventory_yaml is None: + result = jsonify({"message": "No section files found in the values directory to restore from."}) + return result, 400 + + run_write(lambda session: _restore_documents(session, groups_yaml, profiles_yaml, inventory_yaml)) + + changes = ApplyChanges() + job_delay, currently_scheduled = changes.apply_changes() + if job_delay <= 1 and currently_scheduled: + message = "Configuration was restored from section files. There might be previous kubernetes job still " \ + "present in the namespace. Configuration update will be " \ + f"retried {JOB_CREATION_RETRIES} times. If your configuration won't be updated in a few minutes, make sure that " \ + f"snmp-splunk-connect-for-snmp-inventory job isn't present in your kubernetes deployment namespace and " \ + f"click 'Apply changes' button once again." + else: + message = f"Configuration was restored from section files. It will be updated in approximately {job_delay} seconds." + result = jsonify({"message": message}) + return result, 200 + + @apply_changes_blueprint.errorhandler(Exception) def handle_exception(e): current_app.logger.error(traceback.format_exc()) diff --git a/backend/SC4SNMP_UI_backend/auth/routes.py b/backend/SC4SNMP_UI_backend/auth/routes.py index d3774d3..3c74607 100644 --- a/backend/SC4SNMP_UI_backend/auth/routes.py +++ b/backend/SC4SNMP_UI_backend/auth/routes.py @@ -55,4 +55,4 @@ def logout(): def status(): from flask import g - return jsonify({"username": g.current_user}) + return jsonify({"username": g.current_user, "authEnabled": AUTH_ENABLED}) diff --git a/backend/SC4SNMP_UI_backend/common/backend_ui_conversions.py b/backend/SC4SNMP_UI_backend/common/backend_ui_conversions.py index 620ad70..cfd7a49 100644 --- a/backend/SC4SNMP_UI_backend/common/backend_ui_conversions.py +++ b/backend/SC4SNMP_UI_backend/common/backend_ui_conversions.py @@ -270,7 +270,7 @@ def ui2backend(self, document: dict, **kwargs): profiles += ";" result = { 'address': document['address'], - 'port': int(document['port']), + 'port': int(document['port']) if str(document.get('port', "")).strip() else 161, 'version': document['version'], 'community': document['community'], 'secret': document['secret'], diff --git a/backend/SC4SNMP_UI_backend/common/file_to_config_utils.py b/backend/SC4SNMP_UI_backend/common/file_to_config_utils.py new file mode 100644 index 0000000..3310d7b --- /dev/null +++ b/backend/SC4SNMP_UI_backend/common/file_to_config_utils.py @@ -0,0 +1,74 @@ +import csv +import io + + +TRUTHY_VALUES = {"t", "true", "y", "yes", "1"} + + +def str_to_bool(value) -> bool: + """ + Mirrors the truthiness convention used across the codebase for the + section-file/values.yaml CSV fields ("t"/"true"/"y"/"yes"/"1" are truthy, + everything else - including "f"/"false"/"" - is falsy). + """ + return str(value).strip().lower() in TRUTHY_VALUES + + +def groups_yaml_to_documents(groups_dict: dict) -> list: + """ + Inverse of GroupsToYamlDictConversion.convert. Converts a parsed + scheduler.groups section (a dict of {group_name: [hosts...]}, as written + directly to sc4snmp_ui_scheduler_groups.yaml) into the groups_ui Mongo + document shape - one document per group, {group_name: [hosts...]}. + Mongo assigns "_id" on insert, so it's intentionally omitted here. + """ + if not groups_dict: + return [] + return [{group_name: hosts} for group_name, hosts in groups_dict.items()] + + +def profiles_yaml_to_documents(profiles_dict: dict) -> list: + """ + Inverse of ProfilesToYamlDictConversion.convert. Converts a parsed + scheduler.profiles section (a dict of {profile_name: {...}}, as written + directly to sc4snmp_ui_scheduler_profiles.yaml) into the profiles_ui + Mongo document shape - one document per profile, {profile_name: {...}}. + """ + if not profiles_dict: + return [] + return [{profile_name: profile_body} for profile_name, profile_body in profiles_dict.items()] + + +def inventory_csv_to_documents(csv_string: str) -> list: + """ + Inverse of InventoryToYamlDictConversion.convert. Parses the + poller.inventory literal-block CSV (header: address,port,version, + community,secret,security_engine,walk_interval,profiles,smart_profiles, + delete) into the inventory_ui Mongo document shape - one dict per row, + matching InventoryConversion.ui2backend's output fields. + + Rows whose address is blank or starts with "#" are skipped, mirroring + the connector's own convention of allowing commented-out inventory rows. + """ + if not csv_string or not csv_string.strip(): + return [] + + reader = csv.DictReader(io.StringIO(csv_string)) + documents = [] + for row in reader: + address = (row.get("address") or "").strip() + if not address or address.startswith("#"): + continue + documents.append({ + "address": address, + "port": int(row["port"]), + "version": row["version"], + "community": row["community"], + "secret": row["secret"], + "security_engine": row["security_engine"], + "walk_interval": int(row["walk_interval"]), + "profiles": row["profiles"], + "smart_profiles": str_to_bool(row["smart_profiles"]), + "delete": str_to_bool(row["delete"]), + }) + return documents diff --git a/backend/SC4SNMP_UI_backend/common/inventory_utils.py b/backend/SC4SNMP_UI_backend/common/inventory_utils.py index 853a234..5f1906b 100644 --- a/backend/SC4SNMP_UI_backend/common/inventory_utils.py +++ b/backend/SC4SNMP_UI_backend/common/inventory_utils.py @@ -2,7 +2,8 @@ from enum import Enum from typing import Callable from bson import ObjectId -from SC4SNMP_UI_backend.common.backend_ui_conversions import InventoryConversion +from SC4SNMP_UI_backend.common.backend_ui_conversions import InventoryConversion, get_group_or_profile_name_from_backend +from SC4SNMP_UI_backend.common.mongo_utils import run_write mongo_groups = mongo_client.sc4snmp.groups_ui mongo_inventory = mongo_client.sc4snmp.inventory_ui @@ -19,6 +20,19 @@ def get_inventory_type(document): result = "Host" return result +def get_inventory_types_bulk(records): + """ + Resolve Host/Group type for a page of inventory records with a single groups_ui + query instead of one per-record query (see get_inventory_type). Returns a dict + mapping each record's address to "Group" or "Host". + """ + addresses = [record["address"] for record in records] + if not addresses: + return {} + matched_groups = mongo_groups.find({"$or": [{address: {"$exists": True}} for address in addresses]}) + matched_names = {get_group_or_profile_name_from_backend(group) for group in matched_groups} + return {address: ("Group" if address in matched_names else "Host") for address in addresses} + def update_profiles_in_inventory(profile_to_search: str, process_record: Callable, **kwargs): """ When profile is edited, then in some cases inventory records using this profile should be updated. @@ -45,8 +59,9 @@ def __init__(self, mongo_groups, mongo_inventory): self._mongo_groups = mongo_groups self._mongo_inventory = mongo_inventory - def _is_host_in_group(self, address, port) -> (bool, str, str): - groups_from_inventory = list(self._mongo_inventory.find({"address": {"$regex": "^[a-zA-Z].*"}, "delete": False})) + def _is_host_in_group(self, address, port, session=None) -> (bool, str, str): + find_kwargs = {"session": session} if session is not None else {} + groups_from_inventory = list(self._mongo_inventory.find({"address": {"$regex": "^[a-zA-Z].*"}, "delete": False}, **find_kwargs)) break_occurred = False host_in_group = False @@ -58,7 +73,7 @@ def _is_host_in_group(self, address, port) -> (bool, str, str): group_config_name = group_config["address"] group_name = group_config_name group_port = group_config["port"] - group = list(self._mongo_groups.find({group_config_name: {"$exists": 1}})) + group = list(self._mongo_groups.find({group_config_name: {"$exists": 1}}, **find_kwargs)) if len(group) > 0: group = group[0] for i, device in enumerate(group[group_config_name]): @@ -74,9 +89,10 @@ def _is_host_in_group(self, address, port) -> (bool, str, str): return host_in_group, group_id, device_id, group_name - def _is_host_configured(self, address: str, port: str): - existing_inventory_record = list(self._mongo_inventory.find({'address': address, 'port': int(port), "delete": False})) - deleted_inventory_record = list(self._mongo_inventory.find({'address': address, 'port': int(port), "delete": True})) + def _is_host_configured(self, address: str, port: str, session=None): + find_kwargs = {"session": session} if session is not None else {} + existing_inventory_record = list(self._mongo_inventory.find({'address': address, 'port': int(port), "delete": False}, **find_kwargs)) + deleted_inventory_record = list(self._mongo_inventory.find({'address': address, 'port': int(port), "delete": True}, **find_kwargs)) host_configured = False host_configuration = None @@ -88,7 +104,7 @@ def _is_host_configured(self, address: str, port: str): host_configuration = HostConfiguration.SINGLE existing_id_string = str(existing_inventory_record[0]["_id"]) else: - host_in_group, group_id, device_id, group_name = self._is_host_in_group(address, port) + host_in_group, group_id, device_id, group_name = self._is_host_in_group(address, port, session=session) if host_in_group: host_configured = True host_configuration = HostConfiguration.GROUP @@ -96,10 +112,11 @@ def _is_host_configured(self, address: str, port: str): return host_configured, deleted_inventory_record, host_configuration, existing_id_string, group_name - def add_single_host(self, address, port, device_object=None, add: bool=True): + def add_single_host(self, address, port, device_object=None, add: bool=True, session=None): host_configured, deleted_inventory_record, host_configuration, existing_id_string, group_name = \ - self._is_host_configured(address, port) - groups = list(mongo_groups.find({address: {"$exists": True}})) + self._is_host_configured(address, port, session=session) + find_kwargs = {"session": session} if session is not None else {} + groups = list(mongo_groups.find({address: {"$exists": True}}, **find_kwargs)) if host_configured: host_location_message = "in the inventory" if host_configuration == HostConfiguration.SINGLE else \ f"in group {group_name}" @@ -171,6 +188,86 @@ def add_group_host(self, group_name: str, group_id: ObjectId, device_object: dic self._mongo_groups.update_one({"_id": group_id}, new_values) return host_added, message + def add_group_hosts_bulk(self, group_name: str, group_id: ObjectId, device_objects: list): + """ + Adds multiple devices to a group in a single atomic write, instead of looping + add_group_host per device (which would mean N reads + N writes with no + atomicity). Reuses the same uniqueness rules as add_group_host - global + uniqueness across the whole inventory when the group is activated in the + inventory, group-local uniqueness otherwise - plus dedup within the submitted + batch itself, since the DB-based checks can't see sibling rows in the same + request. + + The read, the accept/reject decisions and the write all run inside one + run_write callback against a single snapshot. Deciding against an earlier, + outer read and only re-reading for the write itself would let a concurrent + write - landing after the decision but before the write - go undetected by + the duplicate checks, since they only see what was true when first read. + + :param group_name: name of the group (dynamic dict key in the group document) + :param group_id: ObjectId of the group document + :param device_objects: backend-shaped device dicts (already run through + GroupDeviceConversion.ui2backend), in submission order + :return: list of dicts, one per device_object in the same order: + {"address": str, "port": int|None, "added": bool, "message": str|None} + """ + def _write(session): + group_from_inventory = list( + self._mongo_inventory.find({"address": group_name, "delete": False}, session=session) + ) + grp = list(self._mongo_groups.find({"_id": group_id}, {"_id": 0}, session=session))[0] + in_inventory = len(group_from_inventory) > 0 + + seen_in_batch = set() + accepted = [] + results = [] + + for device_object in device_objects: + address = device_object["address"] + port = str(device_object.get("port", "")) + + if in_inventory: + device_port = port if len(port) > 0 else str(group_from_inventory[0]["port"]) + batch_key = f"{address}:{device_port}" + if batch_key in seen_in_batch: + host_added, message = False, \ + f"Host {address}:{device_port} already exists in the submitted batch. Record was not added." + else: + host_added, message = self.add_single_host(address, device_port, add=False, session=session) + else: + new_device_port = int(port) if len(port) > 0 else -1 + batch_key = f"{address}:{new_device_port}" + if batch_key in seen_in_batch: + host_added, message = False, \ + f"Host {address}:{port} already exists in the submitted batch. Record was not added." + else: + host_added, message = True, None + for existing_device in grp[group_name]: + old_device_port = existing_device.get('port', -1) + if existing_device["address"] == address and old_device_port == new_device_port: + host_added = False + message = f"Host {address}:{port} already exists in group {group_name}. Record was not added." + break + + if host_added: + seen_in_batch.add(batch_key) + accepted.append(device_object) + + results.append({ + "address": address, + "port": device_object.get("port"), + "added": host_added, + "message": message, + }) + + if accepted: + grp[group_name].extend(accepted) + self._mongo_groups.update_one({"_id": group_id}, {"$set": grp}, session=session) + + return results + + return run_write(_write) + def edit_group_host(self, group_name: str, group_id: ObjectId, device_id: str, device_object: dict): group_from_inventory = list(self._mongo_inventory.find({"address": group_name, "delete": False})) group = list(self._mongo_groups.find({"_id": group_id})) diff --git a/backend/SC4SNMP_UI_backend/common/mongo_utils.py b/backend/SC4SNMP_UI_backend/common/mongo_utils.py new file mode 100644 index 0000000..0d14fda --- /dev/null +++ b/backend/SC4SNMP_UI_backend/common/mongo_utils.py @@ -0,0 +1,22 @@ +import os + +from SC4SNMP_UI_backend import mongo_client + + +def transactions_supported(): + return os.getenv("MONGODB_MODE", "standalone").lower() != "standalone" + + +def run_write(write_fn): + """ + Runs write_fn(session) inside a Mongo transaction when the deployment + supports it (MONGODB_MODE != standalone) - atomic, auto-rollback on error. + On standalone, calls write_fn(None): the same writes run sequentially with + no rollback (functional but not atomic). write_fn must forward the session + to every write via session=session for the transactional path to be atomic. + """ + if transactions_supported(): + with mongo_client.start_session() as session: + with session.start_transaction(): + return write_fn(session) + return write_fn(None) diff --git a/backend/SC4SNMP_UI_backend/groups/routes.py b/backend/SC4SNMP_UI_backend/groups/routes.py index 029e68a..3771f5f 100644 --- a/backend/SC4SNMP_UI_backend/groups/routes.py +++ b/backend/SC4SNMP_UI_backend/groups/routes.py @@ -6,6 +6,7 @@ get_group_or_profile_name_from_backend from copy import copy from SC4SNMP_UI_backend.common.inventory_utils import HandleNewDevice, get_inventory_type +from SC4SNMP_UI_backend.common.mongo_utils import run_write groups_blueprint = Blueprint('groups_blueprint', __name__) @@ -15,14 +16,38 @@ mongo_groups = mongo_client.sc4snmp.groups_ui mongo_inventory = mongo_client.sc4snmp.inventory_ui -@groups_blueprint.route('/groups') +@groups_blueprint.route('/groups/count') @login_required -def get_groups_list(): - groups = mongo_groups.find() +def get_groups_count(): + total_count = mongo_groups.count_documents({}) + return jsonify(total_count) + + +@groups_blueprint.route('/groups//') +@login_required +def get_groups_list(page_num, groups_per_page): + page_num = int(page_num) + groups_per_page = int(groups_per_page) + skips = groups_per_page * (page_num - 1) + + # Sorting by _id keeps skip/limit deterministic across refetches (group name is a + # dynamic top-level key, so it can't be sorted on directly); _id order matches the + # existing insertion-order behavior. + page_groups = list(mongo_groups.find().sort("_id", 1).skip(skips).limit(groups_per_page)) + + group_names = [get_group_or_profile_name_from_backend(gr) for gr in page_groups] + # Single batched lookup instead of one query per group: fetch every non-deleted + # inventory address used by this page's groups, then check membership in memory. + in_inventory = set() + if group_names: + in_inventory = { + doc["address"] for doc in + mongo_inventory.find({"address": {"$in": group_names}, "delete": False}, {"address": 1, "_id": 0}) + } + groups_list = [] - for gr in list(groups): - group_name = get_group_or_profile_name_from_backend(gr) - group_in_inventory = True if list(mongo_inventory.find({"address": group_name, "delete": False})) else False + for gr, group_name in zip(page_groups, group_names): + group_in_inventory = group_name in in_inventory groups_list.append(group_conversion.backend2ui(gr, group_in_inventory=group_in_inventory)) return jsonify(groups_list) @@ -74,13 +99,14 @@ def update_group(group_id): def delete_group_and_devices(group_id): group = list(mongo_groups.find({'_id': ObjectId(group_id)}))[0] group_name = get_group_or_profile_name_from_backend(group) - configured_in_inventory = False - with mongo_client.start_session() as session: - with session.start_transaction(): - mongo_groups.delete_one({'_id': ObjectId(group_id)}) - if list(mongo_inventory.find({"address": group_name})): - configured_in_inventory = True - mongo_inventory.update_one({"address": group_name}, {"$set": {"delete": True}}) + + def _delete(session): + mongo_groups.delete_one({'_id': ObjectId(group_id)}, session=session) + configured = bool(list(mongo_inventory.find({"address": group_name}, session=session))) + mongo_inventory.update_one({"address": group_name}, {"$set": {"delete": True}}, session=session) + return configured + + configured_in_inventory = run_write(_delete) if configured_in_inventory: message = f"Group {group_name} was deleted. It was also deleted from the inventory." else: @@ -106,10 +132,11 @@ def get_devices_of_group(group_id, page_num, dev_per_page): group = list(mongo_groups.find({"_id": ObjectId(group_id)}))[0] group_name = get_group_or_profile_name_from_backend(group) - devices_list = [] - for i, device in enumerate(group[group_name]): - devices_list.append(group_device_conversion.backend2ui(device, group_id=group_id, device_id=copy(i))) - devices_list = devices_list[skips:skips+dev_per_page] + # Slice to the requested page before converting, so backend2ui only runs on the + # devices actually returned instead of the whole group's device array every time. + page_devices = list(enumerate(group[group_name]))[skips:skips+dev_per_page] + devices_list = [group_device_conversion.backend2ui(device, group_id=group_id, device_id=copy(i)) + for i, device in page_devices] return jsonify(devices_list) @@ -142,6 +169,57 @@ def add_device_to_group(): return result +@groups_blueprint.route('/devices/add/bulk', methods=['POST']) +@login_required +def add_devices_to_group_bulk(): + payload = request.json or {} + group_id = payload.get("groupId") + devices = payload.get("devices") + if not group_id or not devices: + return jsonify({"message": "groupId and a non-empty devices list are required."}), 400 + + group_records = list(mongo_groups.find({'_id': ObjectId(group_id)}, {"_id": 0})) + if not group_records: + return jsonify({"message": f"Group with id {group_id} was not found."}), 400 + group_name = get_group_or_profile_name_from_backend(group_records[0]) + + # Normalize each device defensively before ui2backend: it does len(document[key]) + # and int(port), which raises on a missing/non-string field - a real risk from + # the paste/CSV adapters rather than the manual grid. A device missing address + # entirely is rejected here rather than crashing ui2backend. + backend_devices = [] + results_by_index = {} + for index, device in enumerate(devices): + address = str(device.get("address") or "").strip() + if not address: + results_by_index[index] = { + "index": index, "address": address, "port": device.get("port"), + "added": False, "message": "Address is required.", + } + continue + normalized = { + "address": address, + "port": str(device.get("port") or ""), + "version": str(device.get("version") or ""), + "community": str(device.get("community") or ""), + "secret": str(device.get("secret") or ""), + "securityEngine": str(device.get("securityEngine") or ""), + } + backend_devices.append((index, group_device_conversion.ui2backend(normalized))) + + handler = HandleNewDevice(mongo_groups, mongo_inventory) + bulk_results = handler.add_group_hosts_bulk(group_name, ObjectId(group_id), [d for _, d in backend_devices]) + + for (original_index, _), result in zip(backend_devices, bulk_results): + results_by_index[original_index] = {"index": original_index, **result} + + ordered_results = [results_by_index[i] for i in range(len(devices))] + added = sum(1 for r in ordered_results if r["added"]) + failed = len(ordered_results) - added + + return jsonify({"added": added, "failed": failed, "results": ordered_results}), 200 + + @groups_blueprint.route('/devices/update/', methods=['POST']) @login_required def update_device_from_group(device_id): diff --git a/backend/SC4SNMP_UI_backend/inventory/routes.py b/backend/SC4SNMP_UI_backend/inventory/routes.py index ed5f31f..389ad6d 100644 --- a/backend/SC4SNMP_UI_backend/inventory/routes.py +++ b/backend/SC4SNMP_UI_backend/inventory/routes.py @@ -3,7 +3,7 @@ from SC4SNMP_UI_backend import mongo_client from SC4SNMP_UI_backend.auth.utils import login_required from SC4SNMP_UI_backend.common.backend_ui_conversions import InventoryConversion -from SC4SNMP_UI_backend.common.inventory_utils import HandleNewDevice, get_inventory_type +from SC4SNMP_UI_backend.common.inventory_utils import HandleNewDevice, get_inventory_type, get_inventory_types_bulk inventory_blueprint = Blueprint('inventory_blueprint', __name__) @@ -19,9 +19,11 @@ def get_inventory_list(page_num, dev_per_page): skips = dev_per_page * (page_num - 1) inventory = list(mongo_inventory.find({"delete": False}).skip(skips).limit(dev_per_page)) + # One batched groups_ui lookup for the whole page instead of one query per row. + inventory_types = get_inventory_types_bulk(inventory) inventory_list = [] for inv in inventory: - inventory_type = get_inventory_type(inv) + inventory_type = inventory_types[inv["address"]] inventory_list.append(inventory_conversion.backend2ui(inv, inventory_type=inventory_type)) return jsonify(inventory_list) diff --git a/backend/requirements.txt b/backend/requirements.txt index d12a6a7..ec183df 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,20 +1,20 @@ -click==8.3.2 +click==8.4.2 Flask==3.1.3 -Flask-Cors==6.0.0 +Flask-Cors==6.0.5 itsdangerous==2.2.0 Jinja2==3.1.6 MarkupSafe==3.0.3 -pymongo==4.6.3 -six==1.16.0 -Werkzeug==3.1.7 -pytest~=8.3 -gunicorn==23.0.0 -kubernetes~=26.1.0 -python-dotenv~=0.21.0 +pymongo==4.17.0 +six==1.17.0 +Werkzeug==3.1.8 +pytest~=9.0, >=9.0.3 +gunicorn==26.1.0 +kubernetes~=36.0.3 +python-dotenv~=1.2, >=1.2.2 PyYAML~=6.0 celery==5.6.3 -redis~=4.5, !=4.5.5 +redis~=8.0 ruamel.yaml===0.17.32 -PyJWT~=2.9 -argon2-cffi~=23.1 +PyJWT~=2.13 +argon2-cffi~=25.1 Flask-Limiter~=3.8 \ No newline at end of file diff --git a/backend/tests/auth/test_auth_endpoints.py b/backend/tests/auth/test_auth_endpoints.py index e0ebdf0..2087c9b 100644 --- a/backend/tests/auth/test_auth_endpoints.py +++ b/backend/tests/auth/test_auth_endpoints.py @@ -55,6 +55,7 @@ def test_status_authenticated(client): response = client.get("/auth/status") assert response.status_code == 200 assert response.json["username"] == TEST_USERNAME + assert response.json["authEnabled"] is True def test_status_unauthenticated(client): diff --git a/backend/tests/common/test_backend_ui_conversions.py b/backend/tests/common/test_backend_ui_conversions.py index b1ef9d6..dd8234a 100644 --- a/backend/tests/common/test_backend_ui_conversions.py +++ b/backend/tests/common/test_backend_ui_conversions.py @@ -335,3 +335,22 @@ def test_inventory_ui_to_backend(self): self.assertDictEqual(inventory_conversion.ui2backend(self.ui_inventory_2, delete=True), back_inv) self.assertRaises(ValueError, inventory_conversion.ui2backend, self.ui_inventory_1) + + def test_inventory_ui_to_backend_empty_port_defaults_to_161(self): + ui_inventory_group_no_port = dict(self.ui_inventory_2) + ui_inventory_group_no_port["port"] = "" + + expected = { + "address": "group_1", + "port": 161, + "version": "2c", + "community": "public", + "secret": "", + "walk_interval": 1900, + "security_engine": "", + "profiles": "prof3", + "smart_profiles": True, + "delete": True + } + + self.assertDictEqual(inventory_conversion.ui2backend(ui_inventory_group_no_port, delete=True), expected) diff --git a/backend/tests/common/test_file_to_config_utils.py b/backend/tests/common/test_file_to_config_utils.py new file mode 100644 index 0000000..863b4f1 --- /dev/null +++ b/backend/tests/common/test_file_to_config_utils.py @@ -0,0 +1,201 @@ +import os +from unittest import TestCase + +import ruamel.yaml +import yaml as pyyaml + +from SC4SNMP_UI_backend.common.file_to_config_utils import ( + groups_yaml_to_documents, + profiles_yaml_to_documents, + inventory_csv_to_documents, +) +from SC4SNMP_UI_backend.apply_changes.config_to_yaml_utils import ( + GroupsToYamlDictConversion, + ProfilesToYamlDictConversion, + InventoryToYamlDictConversion, +) + +REFERENCE_FILES_DIRECTORY = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "../yamls_for_tests/reference_files") + +# Same fixtures as tests/ui_handling/post_endpoints/test_post_apply_changes.py, without +# "_id" since the inverse conversions never produce one - Mongo assigns it on insert. +groups_collection_no_id = [ + { + "group1": [ + {"address": "52.14.243.157", "port": 1163}, + {"address": "20.14.10.0", "port": 161}, + ], + }, + { + "group2": [ + {"address": "0.10.20.30"}, + {"address": "52.14.243.157", "port": 1165, "version": "3", "secret": "mysecret", "security_engine": "aabbccdd1234"}, + ] + } +] + +profiles_collection_no_id = [ + { + "single_metric": { + "frequency": 60, + "varBinds": [['IF-MIB', 'ifMtu', '1']] + } + }, + { + "small_walk": { + "condition": { + "type": "walk" + }, + "varBinds": [['IP-MIB'], ['IF-MIB']] + } + }, + { + "gt_profile": { + "frequency": 10, + "conditions": [ + {"field": "IF-MIB.ifIndex", "operation": "gt", "value": 1} + ], + "varBinds": [['IF-MIB', 'ifOutDiscards']] + } + }, + { + "lt_profile": { + "frequency": 10, + "conditions": [ + {"field": "IF-MIB.ifIndex", "operation": "lt", "value": 2} + ], + "varBinds": [['IF-MIB', 'ifOutDiscards']] + } + }, + { + "in_profile": { + "frequency": 10, + "conditions": [ + {"field": "IF-MIB.ifDescr", "operation": "in", "value": ["eth0", "test value"]} + ], + "varBinds": [['IF-MIB', 'ifOutDiscards']] + } + }, + { + "multiple_conditions": { + "frequency": 10, + "conditions": [ + {"field": "IF-MIB.ifIndex", "operation": "gt", "value": 1}, + {"field": "IF-MIB.ifDescr", "operation": "in", "value": ["eth0", "test value"]} + ], + "varBinds": [['IF-MIB', 'ifOutDiscards'], ['IF-MIB', 'ifOutErrors'], ['IF-MIB', 'ifOutOctets']] + } + } +] + +inventory_collection_no_id = [ + { + "address": "1.1.1.1", + "port": 161, + "version": "2c", + "community": "public", + "secret": "", + "security_engine": "", + "walk_interval": 1800, + "profiles": "small_walk;in_profile", + "smart_profiles": True, + "delete": False + }, + { + "address": "group1", + "port": 1161, + "version": "2c", + "community": "public", + "secret": "", + "security_engine": "", + "walk_interval": 1800, + "profiles": "single_metric;multiple_conditions", + "smart_profiles": False, + "delete": False + } +] + + +class TestFileToConfigUtils(TestCase): + + @classmethod + def setUpClass(cls): + cls.maxDiff = None + + def test_groups_yaml_to_documents(self): + with open(os.path.join(REFERENCE_FILES_DIRECTORY, "scheduler_groups.yaml"), "r") as file: + groups_dict = pyyaml.safe_load(file) + documents = groups_yaml_to_documents(groups_dict) + self.assertEqual(documents, groups_collection_no_id) + + def test_profiles_yaml_to_documents(self): + with open(os.path.join(REFERENCE_FILES_DIRECTORY, "scheduler_profiles.yaml"), "r") as file: + profiles_dict = pyyaml.safe_load(file) + documents = profiles_yaml_to_documents(profiles_dict) + self.assertEqual(documents, profiles_collection_no_id) + + def test_inventory_csv_to_documents(self): + with open(os.path.join(REFERENCE_FILES_DIRECTORY, "poller_inventory.yaml"), "r") as file: + inventory_dict = pyyaml.safe_load(file) + documents = inventory_csv_to_documents(inventory_dict["inventory"]) + self.assertEqual(documents, inventory_collection_no_id) + + def test_inventory_csv_to_documents_skips_commented_and_blank_rows(self): + csv_string = ( + "address,port,version,community,secret,security_engine,walk_interval,profiles,smart_profiles,delete\n" + "1.1.1.1,161,2c,public,,,1800,small_walk,t,f\n" + "#commented_out,161,2c,public,,,1800,small_walk,t,f\n" + ) + documents = inventory_csv_to_documents(csv_string) + self.assertEqual(len(documents), 1) + self.assertEqual(documents[0]["address"], "1.1.1.1") + + def test_inventory_csv_to_documents_empty_string(self): + self.assertEqual(inventory_csv_to_documents(""), []) + self.assertEqual(inventory_csv_to_documents(None), []) + + def test_groups_yaml_to_documents_empty(self): + self.assertEqual(groups_yaml_to_documents({}), []) + self.assertEqual(groups_yaml_to_documents(None), []) + + def test_profiles_yaml_to_documents_empty(self): + self.assertEqual(profiles_yaml_to_documents({}), []) + self.assertEqual(profiles_yaml_to_documents(None), []) + + def test_round_trip_groups(self): + """ + Forward-convert the groups_collection fixture with the existing Mongo->YAML + conversion, dump it exactly as SaveConfigToFileHandler/GroupsTempHandling do, + then read it back with the new inverse conversion and assert the original + documents are recovered. Proves forward and inverse are true inverses and + would catch any ruamel scalar-string/CommentedMap leakage into Mongo. + """ + yaml_dict = GroupsToYamlDictConversion().convert([dict(d) for d in groups_collection_no_id]) + yaml = ruamel.yaml.YAML() + import io + buffer = io.StringIO() + yaml.dump(yaml_dict, buffer) + buffer.seek(0) + round_tripped = pyyaml.safe_load(buffer) + self.assertEqual(groups_yaml_to_documents(round_tripped), groups_collection_no_id) + + def test_round_trip_profiles(self): + yaml_dict = ProfilesToYamlDictConversion().convert([dict(d) for d in profiles_collection_no_id]) + yaml = ruamel.yaml.YAML() + import io + buffer = io.StringIO() + yaml.dump(yaml_dict, buffer) + buffer.seek(0) + round_tripped = pyyaml.safe_load(buffer) + self.assertEqual(profiles_yaml_to_documents(round_tripped), profiles_collection_no_id) + + def test_round_trip_inventory(self): + yaml_dict = InventoryToYamlDictConversion().convert([dict(d) for d in inventory_collection_no_id]) + yaml = ruamel.yaml.YAML() + import io + buffer = io.StringIO() + yaml.dump(yaml_dict, buffer) + buffer.seek(0) + round_tripped = pyyaml.safe_load(buffer) + self.assertEqual(inventory_csv_to_documents(round_tripped["inventory"]), inventory_collection_no_id) diff --git a/backend/tests/ui_handling/get_endpoints/test_get_endpoints.py b/backend/tests/ui_handling/get_endpoints/test_get_endpoints.py index 9f9848e..fae638a 100644 --- a/backend/tests/ui_handling/get_endpoints/test_get_endpoints.py +++ b/backend/tests/ui_handling/get_endpoints/test_get_endpoints.py @@ -88,8 +88,8 @@ def test_get_all_profiles_list(m_client, client): def test_get_groups_list(m_client, client): common_id = "635916b2c8cb7a15f28af40a" - m_client.side_effect = [ - [{ + page_groups = [ + { "_id": common_id, "group_1": [ {"address": "1.2.3.4"} @@ -100,11 +100,16 @@ def test_get_groups_list(m_client, client): "group_2": [ {"address": "1.2.3.4"} ] - }], - [], - [{"address": "group_2"}] + } ] + # mongo_groups.find() is chained with .sort().skip().limit(), so its mocked return + # value needs that chain pre-wired; the batched inventory "$in" lookup that follows + # is a plain, un-chained find() call and is the second side effect. + groups_cursor = mock.MagicMock() + groups_cursor.sort.return_value.skip.return_value.limit.return_value = page_groups + m_client.side_effect = [groups_cursor, [{"address": "group_2"}]] + expected_groups = [ { "_id": common_id, @@ -118,9 +123,21 @@ def test_get_groups_list(m_client, client): } ] - response = client.get('/groups') + response = client.get('/groups/1/50') assert response.json == expected_groups + # Exactly one groups query and one *batched* inventory query, regardless of how many + # groups are on the page - this is the fix for the 1+N query pattern. + assert m_client.call_count == 2 + inventory_call = m_client.call_args_list[1] + assert inventory_call.args == ({"address": {"$in": ["group_1", "group_2"]}, "delete": False}, {"address": 1, "_id": 0}) + + +@mock.patch("pymongo.collection.Collection.count_documents") +def test_get_groups_count(m_client, client): + m_client.return_value = 42 + response = client.get('/groups/count') + assert response.json == 42 @mock.patch("pymongo.collection.Collection.find") @@ -263,9 +280,9 @@ def test_get_devices_of_group(m_client, client): assert response.json == third_result -@mock.patch("SC4SNMP_UI_backend.inventory.routes.get_inventory_type") +@mock.patch("SC4SNMP_UI_backend.common.inventory_utils.mongo_groups") @mock.patch("pymongo.cursor.Cursor.limit") -def test_get_inventory_list(m_cursor, m_get_inventory_type, client): +def test_get_inventory_list(m_cursor, m_groups, client): common_id = "635916b2c8cb7a15f28af40a" m_cursor.side_effect = [ @@ -314,7 +331,12 @@ def test_get_inventory_list(m_cursor, m_get_inventory_type, client): ] ] - m_get_inventory_type.side_effect = ["Host", "Group", "Group"] + # get_inventory_types_bulk runs one groups_ui "$or" query per page instead of one + # get_inventory_type call per row - mock that query's result per page. + m_groups.find.side_effect = [ + [{"_id": common_id, "group_1": [{"address": "1.2.3.4"}]}], + [{"_id": common_id, "group_2": [{"address": "1.2.3.4"}]}], + ] first_result = [ { @@ -363,9 +385,12 @@ def test_get_inventory_list(m_cursor, m_get_inventory_type, client): response = client.get('/inventory/1/2') assert response.json == first_result + # One groups_ui query for the whole 2-row page, not one per row. + assert m_groups.find.call_count == 1 response = client.get('/inventory/2/2') assert response.json == second_result + assert m_groups.find.call_count == 2 @mock.patch("pymongo.collection.Collection.count_documents") diff --git a/backend/tests/ui_handling/post_endpoints/test_post_apply_changes.py b/backend/tests/ui_handling/post_endpoints/test_post_apply_changes.py index 7da516a..9dc15c8 100644 --- a/backend/tests/ui_handling/post_endpoints/test_post_apply_changes.py +++ b/backend/tests/ui_handling/post_endpoints/test_post_apply_changes.py @@ -338,10 +338,74 @@ def test_apply_changes_job_currently_scheduled_job_present_in_namespace(m_find, m_get_job_config.return_value = ("val2", "val1") m_create_job.side_effect = ApiException() - response = client.post("/apply-changes") + inspect_mock = Mock() + inspect_mock.scheduled.return_value = {"worker1": [{"request": {"id": "test_id"}}]} + fake_celery = Mock(control=Mock(inspect=Mock(return_value=inspect_mock))) + with mock.patch.dict(client.application.extensions, {"celery": fake_celery}): + response = client.post("/apply-changes") m_find.assert_has_calls(calls_find) m_create_job.assert_has_calls(create_job_calls) assert not m_run_job.apply_async.called assert response.json == {"message": "Configuration will be updated in approximately 130 seconds."} delete_generated_files() reset_generated_values() + + +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.VALUES_DIRECTORY", VALUES_TEST_DIRECTORY) +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.TMP_DIR", VALUES_TEST_DIRECTORY) +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.datetime") +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.create_job") +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.get_job_config") +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.run_job") +@mock.patch("pymongo.collection.Collection.update_one") +@mock.patch("pymongo.collection.Collection.find") +def test_apply_changes_stale_currently_scheduled_is_reset_and_rescheduled(m_find, m_update, m_run_job, m_get_job_config, m_create_job, m_datetime, client): + """ + currently_scheduled can get stuck True in mongo if Redis (the Celery broker) was reset + independently of this backend process - e.g. only Redis was wiped during an environment + rebuild. In that case the recorded task_id is no longer known to any worker, so + ScheduleHandler must treat the stuck flag as stale, reset it, and schedule a fresh job + instead of leaving apply-changes permanently blocked. + """ + datetime_object_old = datetime.datetime(2020, 7, 10, 10, 27, 10, 0) + datetime_object_new = datetime.datetime(2020, 7, 10, 10, 30, 0, 0) + m_datetime.datetime.utcnow = mock.Mock(return_value=datetime_object_new) + collection = { + "_id": ObjectId(common_id), + "previous_job_start_time": datetime_object_old, + "currently_scheduled": True, + "task_id": "stale_task_id" + } + m_find.side_effect = [ + groups_collection, # call from SaveConfigToFileHandler + profiles_collection, # call from SaveConfigToFileHandler + inventory_collection, # call from SaveConfigToFileHandler + [collection], # call from CheckJobHandler + [collection], # call from ScheduleHandler + ] + create_job_calls = [ + call("val1", "val2", "sc4snmp") + ] + m_get_job_config.return_value = ("val2", "val1") + m_create_job.side_effect = ApiException() + + apply_async_result = Mock() + apply_async_result.id = "new_task_id" + m_run_job.apply_async.return_value = apply_async_result + m_update.return_value = None + + inspect_mock = Mock() + inspect_mock.scheduled.return_value = {} + fake_celery = Mock(control=Mock(inspect=Mock(return_value=inspect_mock))) + with mock.patch.dict(client.application.extensions, {"celery": fake_celery}): + response = client.post("/apply-changes") + + m_create_job.assert_has_calls(create_job_calls) + reset_call = call({"_id": ObjectId(common_id)}, {"$set": {"currently_scheduled": False, "task_id": None}}) + reschedule_call = call({"_id": ObjectId(common_id)}, {"$set": {"currently_scheduled": True, "task_id": "new_task_id"}}) + assert reset_call in m_update.call_args_list + assert reschedule_call in m_update.call_args_list + m_run_job.apply_async.assert_called_with(countdown=130, queue='apply_changes') + assert response.json == {"message": "Configuration will be updated in approximately 130 seconds."} + delete_generated_files() + reset_generated_values() diff --git a/backend/tests/ui_handling/post_endpoints/test_post_groups.py b/backend/tests/ui_handling/post_endpoints/test_post_groups.py index 7d06d0b..9677370 100644 --- a/backend/tests/ui_handling/post_endpoints/test_post_groups.py +++ b/backend/tests/ui_handling/post_endpoints/test_post_groups.py @@ -148,15 +148,15 @@ def test_update_group_record_with_name_existing_in_inventory_as_hostname_failure @mock.patch("pymongo.collection.Collection.find") @mock.patch("pymongo.collection.Collection.delete_one") @mock.patch("pymongo.collection.Collection.update_one") -@mock.patch("pymongo.MongoClient.start_session") -def test_delete_group_and_devices(m_session, m_update, m_delete, m_find, client): +def test_delete_group_and_devices(m_update, m_delete, m_find, client): + # conftest.py sets MONGODB_MODE=standalone, so this exercises the + # non-transactional fallback path (no session threaded through writes). backend_group = { "_id": ObjectId(common_id), "group_1": [ {"address": "1.2.3.4"}, ] } - m_session.return_value.__enter__.return_value.start_transaction.__enter__ = Mock() m_find.side_effect = [ [backend_group], @@ -165,7 +165,7 @@ def test_delete_group_and_devices(m_session, m_update, m_delete, m_find, client) calls_find = [ call({'_id': ObjectId(common_id)}), - call({"address": "group_1"}) + call({"address": "group_1"}, session=None) ] m_delete.return_value = None @@ -173,8 +173,8 @@ def test_delete_group_and_devices(m_session, m_update, m_delete, m_find, client) response = client.post(f"/groups/delete/{common_id}") m_find.assert_has_calls(calls_find) - assert m_delete.call_args == call({'_id': ObjectId(common_id)}) - assert m_update.call_args == call({"address": "group_1"}, {"$set": {"delete": True}}) + assert m_delete.call_args == call({'_id': ObjectId(common_id)}, session=None) + assert m_update.call_args == call({"address": "group_1"}, {"$set": {"delete": True}}, session=None) assert response.json == { "message": "Group group_1 was deleted."} @@ -185,8 +185,44 @@ def test_delete_group_and_devices(m_session, m_update, m_delete, m_find, client) response = client.post(f"/groups/delete/{common_id}") m_find.assert_has_calls(calls_find) - assert m_delete.call_args == call({'_id': ObjectId(common_id)}) - assert m_update.call_args == call({"address": "group_1"}, {"$set": {"delete": True}}) + assert m_delete.call_args == call({'_id': ObjectId(common_id)}, session=None) + assert m_update.call_args == call({"address": "group_1"}, {"$set": {"delete": True}}, session=None) + assert response.json == { + "message": "Group group_1 was deleted. It was also deleted from the inventory."} + + +@mock.patch("pymongo.collection.Collection.find") +@mock.patch("pymongo.collection.Collection.delete_one") +@mock.patch("pymongo.collection.Collection.update_one") +@mock.patch("pymongo.MongoClient.start_session") +def test_delete_group_and_devices_uses_transaction_in_replication_mode(m_session, m_update, m_delete, m_find, + client, monkeypatch): + # When the deployment is a replica set, the writes must run inside a + # Mongo transaction (session threaded through every write). + monkeypatch.setenv("MONGODB_MODE", "replication") + + backend_group = { + "_id": ObjectId(common_id), + "group_1": [ + {"address": "1.2.3.4"}, + ] + } + m_session.return_value.__enter__.return_value.start_transaction.__enter__ = Mock() + + m_find.side_effect = [ + [backend_group], + [{}] + ] + + m_delete.return_value = None + m_update.return_value = None + + response = client.post(f"/groups/delete/{common_id}") + + session = m_session.return_value.__enter__.return_value + assert call({"address": "group_1"}, session=session) in m_find.call_args_list + assert m_delete.call_args == call({'_id': ObjectId(common_id)}, session=session) + assert m_update.call_args == call({"address": "group_1"}, {"$set": {"delete": True}}, session=session) assert response.json == { "message": "Group group_1 was deleted. It was also deleted from the inventory."} @@ -576,4 +612,222 @@ def test_delete_device_from_group_record(m_find, m_update, client): response = client.post(f"/devices/delete/{common_id}-0") assert m_find.call_args == call({'_id': ObjectId(common_id)}, {"_id": 0}) assert m_update.call_args == call({"_id": ObjectId(common_id)}, {"$set": backend_group_new2}) - assert response.json == {'message': 'Device 1.1.1.1: from group group_1 was deleted.'} \ No newline at end of file + assert response.json == {'message': 'Device 1.1.1.1: from group group_1 was deleted.'} + + +# TEST ADDING DEVICES IN BULK +ui_bulk_devices_success = lambda: { + "groupId": str(common_id), + "devices": [ + {"address": "2.2.2.2", "port": "", "version": "3", "community": "", "secret": "snmpv3", "securityEngine": ""}, + {"address": "3.3.3.3", "port": "162", "version": "2c", "community": "public", "secret": "", "securityEngine": ""}, + ] + } + +backend_group_bulk_success_new = lambda: { + "_id": ObjectId(common_id), + "group_1": [ + {"address": "1.2.3.4", "port": 161}, + {"address": "2.2.2.2", "version": "3", "secret": "snmpv3"}, + {"address": "3.3.3.3", "port": 162, "version": "2c", "community": "public"}, + ] + } + +@mock.patch("pymongo.collection.Collection.update_one") +@mock.patch("pymongo.collection.Collection.find") +def test_add_devices_to_group_bulk_not_configured_in_inventory_success(m_find, m_update, client): + m_find.side_effect = [ + [backend_group_add_device_old()], # call from group/routes.add_devices_to_group_bulk + [], # call from HandleNewDevice.add_group_hosts_bulk._write (group_from_inventory) + [backend_group_add_device_old()], # call from HandleNewDevice.add_group_hosts_bulk._write (group read) + ] + calls_find = [ + call({'_id': ObjectId(common_id)}, {"_id": 0}), + call({"address": "group_1", "delete": False}, session=None), + call({'_id': ObjectId(common_id)}, {"_id": 0}, session=None), + ] + m_update.return_value = None + + response = client.post(f"/devices/add/bulk", json=ui_bulk_devices_success()) + m_find.assert_has_calls(calls_find) + assert m_update.call_args == call({"_id": ObjectId(common_id)}, {"$set": backend_group_bulk_success_new()}, session=None) + assert response.json == { + "added": 2, + "failed": 0, + "results": [ + {"index": 0, "address": "2.2.2.2", "port": None, "added": True, "message": None}, + {"index": 1, "address": "3.3.3.3", "port": 162, "added": True, "message": None}, + ], + } + + +@mock.patch("pymongo.collection.Collection.update_one") +@mock.patch("pymongo.collection.Collection.find") +def test_add_devices_to_group_bulk_configured_in_inventory_partial_success(m_find, m_update, client): + ui_devices = { + "groupId": str(common_id), + "devices": [ + {"address": "2.2.2.2", "port": "", "version": "3", "community": "", "secret": "snmpv3", "securityEngine": ""}, + {"address": "5.5.5.5", "port": "161", "version": "3", "community": "", "secret": "snmpv3", "securityEngine": ""}, + ] + } + + existing_device_inventory = { + "_id": ObjectId(common_id), + "address": "5.5.5.5", + "port": 161, + "version": "2c", + "community": "public", + "secret": "", + "walk_interval": 1800, + "security_engine": "", + "profiles": "prof1", + "smart_profiles": False, + "delete": False + } + + m_find.side_effect = [ + [backend_group_add_device_old()], # call from group/routes.add_devices_to_group_bulk + [group_inventory()], # call from HandleNewDevice.add_group_hosts_bulk._write (group_from_inventory) + [backend_group_add_device_old()], # call from HandleNewDevice.add_group_hosts_bulk._write (group read) + [], # device 2.2.2.2: HandleNewDevice._is_host_configured + [], # device 2.2.2.2: HandleNewDevice._is_host_configured + [group_inventory()], # device 2.2.2.2: HandleNewDevice._is_host_in_group + [backend_group_add_device_old()], # device 2.2.2.2: HandleNewDevice._is_host_in_group + [], # device 2.2.2.2: HandleNewDevice.add_single_host + [existing_device_inventory], # device 5.5.5.5: HandleNewDevice._is_host_configured + [], # device 5.5.5.5: HandleNewDevice._is_host_configured + [], # device 5.5.5.5: HandleNewDevice.add_single_host + ] + calls_find = [ + call({'_id': ObjectId(common_id)}, {"_id": 0}), + call({"address": "group_1", "delete": False}, session=None), + call({'_id': ObjectId(common_id)}, {"_id": 0}, session=None), + call({'address': "2.2.2.2", 'port': 1161, "delete": False}), + call({'address': "2.2.2.2", 'port': 1161, "delete": True}), + call({"address": {"$regex": "^[a-zA-Z].*"}, "delete": False}), + call({"group_1": {"$exists": 1}}), + call({"2.2.2.2": {"$exists": True}}), + call({'address': "5.5.5.5", 'port': 161, "delete": False}), + call({'address': "5.5.5.5", 'port': 161, "delete": True}), + call({"5.5.5.5": {"$exists": True}}), + ] + m_update.return_value = None + + response = client.post(f"/devices/add/bulk", json=ui_devices) + m_find.assert_has_calls(calls_find) + assert m_update.call_args == call({"_id": ObjectId(common_id)}, {"$set": { + "_id": ObjectId(common_id), + "group_1": [ + {"address": "1.2.3.4", "port": 161}, + {"address": "2.2.2.2", "version": "3", "secret": "snmpv3"}, + ] + }}, session=None) + assert response.json == { + "added": 1, + "failed": 1, + "results": [ + {"index": 0, "address": "2.2.2.2", "port": None, "added": True, "message": None}, + {"index": 1, "address": "5.5.5.5", "port": 161, "added": False, + "message": "Host 5.5.5.5:161 already exists in the inventory. Record was not added."}, + ], + } + + +@mock.patch("pymongo.collection.Collection.update_one") +@mock.patch("pymongo.collection.Collection.find") +def test_add_devices_to_group_bulk_in_batch_duplicate(m_find, m_update, client): + device = {"address": "9.9.9.9", "port": "162", "version": "2c", "community": "public", "secret": "", "securityEngine": ""} + ui_devices = { + "groupId": str(common_id), + "devices": [device, dict(device)] + } + + m_find.side_effect = [ + [backend_group_add_device_old()], # call from group/routes.add_devices_to_group_bulk + [], # call from HandleNewDevice.add_group_hosts_bulk._write (group_from_inventory, not in inventory) + [backend_group_add_device_old()], # call from HandleNewDevice.add_group_hosts_bulk._write (group read) + ] + calls_find = [ + call({'_id': ObjectId(common_id)}, {"_id": 0}), + call({"address": "group_1", "delete": False}, session=None), + call({'_id': ObjectId(common_id)}, {"_id": 0}, session=None), + ] + m_update.return_value = None + + response = client.post(f"/devices/add/bulk", json=ui_devices) + m_find.assert_has_calls(calls_find) + assert m_update.call_args == call({"_id": ObjectId(common_id)}, {"$set": { + "_id": ObjectId(common_id), + "group_1": [ + {"address": "1.2.3.4", "port": 161}, + {"address": "9.9.9.9", "port": 162, "version": "2c", "community": "public"}, + ] + }}, session=None) + assert response.json == { + "added": 1, + "failed": 1, + "results": [ + {"index": 0, "address": "9.9.9.9", "port": 162, "added": True, "message": None}, + {"index": 1, "address": "9.9.9.9", "port": 162, "added": False, + "message": "Host 9.9.9.9:162 already exists in the submitted batch. Record was not added."}, + ], + } + + +@mock.patch("pymongo.collection.Collection.update_one") +@mock.patch("pymongo.collection.Collection.find") +def test_add_devices_to_group_bulk_missing_group_id_failure(m_find, m_update, client): + response = client.post(f"/devices/add/bulk", json={"devices": [{"address": "1.1.1.1"}]}) + assert not m_find.called + assert not m_update.called + assert response.status_code == 400 + assert response.json == {"message": "groupId and a non-empty devices list are required."} + + +@mock.patch("pymongo.collection.Collection.update_one") +@mock.patch("pymongo.collection.Collection.find") +def test_add_devices_to_group_bulk_empty_devices_failure(m_find, m_update, client): + response = client.post(f"/devices/add/bulk", json={"groupId": str(common_id), "devices": []}) + assert not m_find.called + assert not m_update.called + assert response.status_code == 400 + assert response.json == {"message": "groupId and a non-empty devices list are required."} + + +@mock.patch("pymongo.collection.Collection.find") +@mock.patch("pymongo.collection.Collection.update_one") +@mock.patch("pymongo.MongoClient.start_session") +def test_add_devices_to_group_bulk_uses_transaction_in_replication_mode(m_session, m_update, m_find, client, monkeypatch): + # When the deployment is a replica set, the final write inside add_group_hosts_bulk + # must run inside a Mongo transaction (session threaded through find + update_one). + monkeypatch.setenv("MONGODB_MODE", "replication") + m_session.return_value.__enter__.return_value.start_transaction.__enter__ = Mock() + + ui_devices = { + "groupId": str(common_id), + "devices": [ + {"address": "2.2.2.2", "port": "", "version": "3", "community": "", "secret": "snmpv3", "securityEngine": ""}, + ] + } + + m_find.side_effect = [ + [backend_group_add_device_old()], # call from group/routes.add_devices_to_group_bulk + [], # call from HandleNewDevice.add_group_hosts_bulk._write (group_from_inventory, session=session) + [backend_group_add_device_old()], # call from HandleNewDevice.add_group_hosts_bulk._write (group read, session=session) + ] + m_update.return_value = None + + response = client.post(f"/devices/add/bulk", json=ui_devices) + + session = m_session.return_value.__enter__.return_value + assert call({"address": "group_1", "delete": False}, session=session) in m_find.call_args_list + assert call({'_id': ObjectId(common_id)}, {"_id": 0}, session=session) in m_find.call_args_list + assert m_update.call_args == call({"_id": ObjectId(common_id)}, {"$set": backend_group_add_device_success_new()}, session=session) + assert response.json == { + "added": 1, + "failed": 0, + "results": [ + {"index": 0, "address": "2.2.2.2", "port": None, "added": True, "message": None}, + ], + } \ No newline at end of file diff --git a/backend/tests/ui_handling/post_endpoints/test_post_inventory.py b/backend/tests/ui_handling/post_endpoints/test_post_inventory.py index ea28e2b..c5298b1 100644 --- a/backend/tests/ui_handling/post_endpoints/test_post_inventory.py +++ b/backend/tests/ui_handling/post_endpoints/test_post_inventory.py @@ -660,6 +660,47 @@ def test_add_group_success(m_find, m_insert, m_delete, client): assert response.json == "success" +@mock.patch("pymongo.collection.Collection.delete_one") +@mock.patch("pymongo.collection.Collection.insert_one") +@mock.patch("pymongo.collection.Collection.find") +def test_add_group_with_empty_port_defaults_to_161(m_find, m_insert, m_delete, client): + # A Group-type inventory record can be submitted with no port at all (the UI no + # longer requires it); the backend should treat it as 161, same as SC4SNMP core does. + m_insert.return_value = None + m_delete.return_value = None + + new_group_ui_no_port = dict(new_group_ui_inventory()) + new_group_ui_no_port["port"] = "" + + m_find.side_effect = [ + [], # call from HandleNewDevice.add_group_to_inventory + [], # call from HandleNewDevice.add_group_to_inventory + [new_group_backend()], # call from HandleNewDevice.add_group_to_inventory + [], # call from HandleNewDevice._is_host_configured + [], # call from HandleNewDevice._is_host_configured + [existing_group_inventory_backend()], # call from HandleNewDevice._is_host_in_group + [existing_group_backend()], # call from HandleNewDevice._is_host_in_group + [], # call from HandleNewDevice.add_single_host + ] + + calls_find = [ + call({'address': "group_1", "delete": False}), # call from HandleNewDevice.add_group_to_inventory + call({'address': "group_1", "delete": True}), # call from HandleNewDevice.add_group_to_inventory + call({'group_1': {"$exists": 1}}), # call from HandleNewDevice.add_group_to_inventory + call({'address': "1.2.3.4", 'port': 161, "delete": False}), # call from HandleNewDevice._is_host_configured + call({'address': "1.2.3.4", 'port': 161, "delete": True}), # call from HandleNewDevice._is_host_configured + call({"address": {"$regex": "^[a-zA-Z].*"}, "delete": False}), # call from HandleNewDevice._is_host_in_group + call({"group_2": {"$exists": 1}}), # call from HandleNewDevice._is_host_in_group + call({'1.2.3.4': {"$exists": True}}), # call from HandleNewDevice.add_single_host + ] + + response = client.post(f"/inventory/add", json=new_group_ui_no_port) + m_find.assert_has_calls(calls_find) + assert m_insert.call_args == call(new_group_backend_inventory()) # stored port is 161 + assert not m_delete.called + assert response.json == "success" + + @mock.patch("pymongo.collection.Collection.delete_one") @mock.patch("pymongo.collection.Collection.insert_one") @mock.patch("pymongo.collection.Collection.find") diff --git a/backend/tests/ui_handling/post_endpoints/test_post_load_config.py b/backend/tests/ui_handling/post_endpoints/test_post_load_config.py new file mode 100644 index 0000000..9455284 --- /dev/null +++ b/backend/tests/ui_handling/post_endpoints/test_post_load_config.py @@ -0,0 +1,277 @@ +import datetime +import os +import shutil +from unittest import mock +from unittest.mock import call, Mock + +import pytest +from bson import ObjectId + +from SC4SNMP_UI_backend.apply_changes.apply_changes import SingletonMeta +from SC4SNMP_UI_backend.apply_changes import handling_chain +from SC4SNMP_UI_backend.apply_changes.handling_chain import TMP_FILE_PREFIX + +REFERENCE_FILES_DIRECTORY = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "../../yamls_for_tests/reference_files") + +common_id = "635916b2c8cb7a15f28af40a" + +# Documents expected to be parsed out of the reference section files (mirrors +# tests/ui_handling/post_endpoints/test_post_apply_changes.py's fixtures, minus "_id" +# since the inverse conversions never produce one). +groups_collection_no_id = [ + { + "group1": [ + {"address": "52.14.243.157", "port": 1163}, + {"address": "20.14.10.0", "port": 161}, + ], + }, + { + "group2": [ + {"address": "0.10.20.30"}, + {"address": "52.14.243.157", "port": 1165, "version": "3", "secret": "mysecret", "security_engine": "aabbccdd1234"}, + ] + } +] + +profiles_collection_no_id = [ + {"single_metric": {"frequency": 60, "varBinds": [['IF-MIB', 'ifMtu', '1']]}}, + {"small_walk": {"condition": {"type": "walk"}, "varBinds": [['IP-MIB'], ['IF-MIB']]}}, + {"gt_profile": {"frequency": 10, "conditions": [{"field": "IF-MIB.ifIndex", "operation": "gt", "value": 1}], + "varBinds": [['IF-MIB', 'ifOutDiscards']]}}, + {"lt_profile": {"frequency": 10, "conditions": [{"field": "IF-MIB.ifIndex", "operation": "lt", "value": 2}], + "varBinds": [['IF-MIB', 'ifOutDiscards']]}}, + {"in_profile": {"frequency": 10, + "conditions": [{"field": "IF-MIB.ifDescr", "operation": "in", "value": ["eth0", "test value"]}], + "varBinds": [['IF-MIB', 'ifOutDiscards']]}}, + {"multiple_conditions": { + "frequency": 10, + "conditions": [ + {"field": "IF-MIB.ifIndex", "operation": "gt", "value": 1}, + {"field": "IF-MIB.ifDescr", "operation": "in", "value": ["eth0", "test value"]} + ], + "varBinds": [['IF-MIB', 'ifOutDiscards'], ['IF-MIB', 'ifOutErrors'], ['IF-MIB', 'ifOutOctets']] + }}, +] + +inventory_collection_no_id = [ + { + "address": "1.1.1.1", "port": 161, "version": "2c", "community": "public", "secret": "", + "security_engine": "", "walk_interval": 1800, "profiles": "small_walk;in_profile", + "smart_profiles": True, "delete": False + }, + { + "address": "group1", "port": 1161, "version": "2c", "community": "public", "secret": "", + "security_engine": "", "walk_interval": 1800, "profiles": "single_metric;multiple_conditions", + "smart_profiles": False, "delete": False + } +] + +config_record = { + "_id": ObjectId(common_id), + "previous_job_start_time": None, + "currently_scheduled": False, + "task_id": None +} + + +@pytest.fixture(autouse=True) +def reset_singleton(): + yield + SingletonMeta._instances = {} + + +def _write_section_files(directory): + """ + Copies the reference section-file fixtures into `directory` under the + sc4snmp_ui_
.yaml naming /load-config expects, so the route reads + the same fixture content used elsewhere for the Mongo<->YAML round trip. + """ + for file_name in ("scheduler_groups.yaml", "scheduler_profiles.yaml", "poller_inventory.yaml"): + shutil.copy( + os.path.join(REFERENCE_FILES_DIRECTORY, file_name), + os.path.join(directory, f"{TMP_FILE_PREFIX}{file_name}"), + ) + + +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.VALUES_FILE", "") +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.KEEP_TEMP_FILES", "true") +@mock.patch("datetime.datetime") +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.create_job") +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.get_job_config") +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.run_job") +@mock.patch("pymongo.collection.Collection.delete_many") +@mock.patch("pymongo.collection.Collection.insert_many") +@mock.patch("pymongo.collection.Collection.update_one") +@mock.patch("pymongo.collection.Collection.find") +def test_load_config_restores_from_section_files(m_find, m_update, m_insert_many, m_delete_many, m_run_job, + m_get_job_config, m_create_job, m_datetime, + client, tmp_path, monkeypatch): + # conftest.py sets MONGODB_MODE=standalone, so this exercises the + # non-transactional fallback path (no session threaded through writes). + monkeypatch.setattr(handling_chain, "VALUES_DIRECTORY", str(tmp_path)) + monkeypatch.setattr(handling_chain, "TMP_DIR", str(tmp_path)) + _write_section_files(tmp_path) + + datetime_object = datetime.datetime(2020, 7, 10, 10, 30, 0, 0) + m_datetime.utcnow = mock.Mock(return_value=datetime_object) + + m_find.side_effect = [ + [], # mongo_inventory.find({"delete": False}) - reconciliation read, no existing rows + groups_collection_no_id, # mongo_groups.find() from SaveConfigToFileHandler + profiles_collection_no_id, # mongo_profiles.find() from SaveConfigToFileHandler + inventory_collection_no_id, # mongo_inventory.find() from SaveConfigToFileHandler + [config_record], # mongo_config_collection.find() from CheckJobHandler + [config_record], # mongo_config_collection.find() from ScheduleHandler + ] + m_get_job_config.return_value = ("val2", "val1") + m_create_job.return_value = None + m_update.return_value = None + m_insert_many.return_value = None + m_delete_many.return_value = None + + response = client.post("/load-config") + + m_delete_many.assert_has_calls([call({}, session=None), call({}, session=None)]) + m_insert_many.assert_has_calls([ + call(groups_collection_no_id, session=None), + call(profiles_collection_no_id, session=None), + ]) + + reconciliation_find_call = call({"delete": False}, session=None) + assert reconciliation_find_call in m_find.call_args_list + + upsert_calls = [ + call({"address": doc["address"], "port": doc["port"]}, {"$set": doc}, upsert=True, session=None) + for doc in inventory_collection_no_id + ] + m_update.assert_has_calls(upsert_calls) + + assert response.status_code == 200 + assert response.json == { + "message": "Configuration was restored from section files. It will be updated in approximately 1 seconds." + } + + +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.VALUES_FILE", "") +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.KEEP_TEMP_FILES", "true") +@mock.patch("datetime.datetime") +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.create_job") +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.get_job_config") +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.run_job") +@mock.patch("pymongo.collection.Collection.delete_many") +@mock.patch("pymongo.collection.Collection.insert_many") +@mock.patch("pymongo.collection.Collection.update_one") +@mock.patch("pymongo.collection.Collection.find") +@mock.patch("pymongo.MongoClient.start_session") +def test_load_config_restores_from_section_files_uses_transaction_in_replication_mode( + m_session, m_find, m_update, m_insert_many, m_delete_many, m_run_job, + m_get_job_config, m_create_job, m_datetime, client, tmp_path, monkeypatch): + # When the deployment is a replica set, the writes must run inside a + # Mongo transaction (session threaded through every write) so a + # mid-restore failure rolls back instead of leaving partial state. + monkeypatch.setenv("MONGODB_MODE", "replication") + monkeypatch.setattr(handling_chain, "VALUES_DIRECTORY", str(tmp_path)) + monkeypatch.setattr(handling_chain, "TMP_DIR", str(tmp_path)) + _write_section_files(tmp_path) + + datetime_object = datetime.datetime(2020, 7, 10, 10, 30, 0, 0) + m_datetime.utcnow = mock.Mock(return_value=datetime_object) + + m_session.return_value.__enter__.return_value.start_transaction.__enter__ = Mock() + + m_find.side_effect = [ + [], # mongo_inventory.find({"delete": False}) - reconciliation read, no existing rows + groups_collection_no_id, # mongo_groups.find() from SaveConfigToFileHandler + profiles_collection_no_id, # mongo_profiles.find() from SaveConfigToFileHandler + inventory_collection_no_id, # mongo_inventory.find() from SaveConfigToFileHandler + [config_record], # mongo_config_collection.find() from CheckJobHandler + [config_record], # mongo_config_collection.find() from ScheduleHandler + ] + m_get_job_config.return_value = ("val2", "val1") + m_create_job.return_value = None + m_update.return_value = None + m_insert_many.return_value = None + m_delete_many.return_value = None + + response = client.post("/load-config") + + session = m_session.return_value.__enter__.return_value + m_delete_many.assert_has_calls([call({}, session=session), call({}, session=session)]) + m_insert_many.assert_has_calls([ + call(groups_collection_no_id, session=session), + call(profiles_collection_no_id, session=session), + ]) + + reconciliation_find_call = call({"delete": False}, session=session) + assert reconciliation_find_call in m_find.call_args_list + + upsert_calls = [ + call({"address": doc["address"], "port": doc["port"]}, {"$set": doc}, upsert=True, session=session) + for doc in inventory_collection_no_id + ] + m_update.assert_has_calls(upsert_calls) + + assert response.status_code == 200 + assert response.json == { + "message": "Configuration was restored from section files. It will be updated in approximately 1 seconds." + } + + +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.VALUES_FILE", "") +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.KEEP_TEMP_FILES", "true") +@mock.patch("datetime.datetime") +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.create_job") +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.get_job_config") +@mock.patch("SC4SNMP_UI_backend.apply_changes.handling_chain.run_job") +@mock.patch("pymongo.collection.Collection.delete_many") +@mock.patch("pymongo.collection.Collection.insert_many") +@mock.patch("pymongo.collection.Collection.update_one") +@mock.patch("pymongo.collection.Collection.find") +def test_load_config_soft_deletes_hosts_missing_from_files(m_find, m_update, m_insert_many, m_delete_many, m_run_job, + m_get_job_config, m_create_job, m_datetime, + client, tmp_path, monkeypatch): + monkeypatch.setattr(handling_chain, "VALUES_DIRECTORY", str(tmp_path)) + monkeypatch.setattr(handling_chain, "TMP_DIR", str(tmp_path)) + _write_section_files(tmp_path) + + datetime_object = datetime.datetime(2020, 7, 10, 10, 30, 0, 0) + m_datetime.utcnow = mock.Mock(return_value=datetime_object) + + orphan_id = ObjectId("635916b2c8cb7a15f28af40b") + orphan_record = {"_id": orphan_id, "address": "orphan_host", "port": 9999, "delete": False} + + m_find.side_effect = [ + [orphan_record], # mongo_inventory.find({"delete": False}) - one host not present in files + groups_collection_no_id, + profiles_collection_no_id, + inventory_collection_no_id, + [config_record], + [config_record], + ] + m_get_job_config.return_value = ("val2", "val1") + m_create_job.return_value = None + m_update.return_value = None + m_insert_many.return_value = None + m_delete_many.return_value = None + + response = client.post("/load-config") + + soft_delete_call = call({"_id": orphan_id}, {"$set": {"delete": True}}, session=None) + assert soft_delete_call in m_update.call_args_list + + upsert_calls = [ + call({"address": doc["address"], "port": doc["port"]}, {"$set": doc}, upsert=True, session=None) + for doc in inventory_collection_no_id + ] + m_update.assert_has_calls(upsert_calls) + assert response.status_code == 200 + + +def test_load_config_returns_400_when_no_section_files_present(client, tmp_path, monkeypatch): + monkeypatch.setattr(handling_chain, "VALUES_DIRECTORY", str(tmp_path)) + + response = client.post("/load-config") + + assert response.status_code == 400 + assert response.json == {"message": "No section files found in the values directory to restore from."} diff --git a/frontend/lerna.json b/frontend/lerna.json index c022e8e..7bf5e99 100644 --- a/frontend/lerna.json +++ b/frontend/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "1.2.2", + "version": "1.2.3-beta.8", "command": { "publish": { "ignoreChanges": ["*.md"] diff --git a/frontend/package.json b/frontend/package.json index 2f9b4e8..0842e17 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -22,18 +22,30 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" }, "dependencies": { - "cors": "^2.8.5" + "axios": "^1.16.0", + "cors": "^2.8.5", + "uuid": "^14.0.0" }, "resolutions": { "uuid": "^14.0.0", "follow-redirects": "^1.16.0", "axios": "^1.16.0", - "undici": "^6.27.0", + "undici": "^8.10.0", "tmp": "^0.2.6", "form-data": "^4.0.6", - "brace-expansion": "^5.0.6", + "brace-expansion": "^5.0.9", "yaml": "^2.8.3", - "tar": "^7.5.16", - "js-yaml": "^4.2.0" + "tar": "^7.5.21", + "js-yaml": "^5.3.0", + "postcss": "^8.5.23", + "nanoid": "^6.0.1", + "ip-address": "^10.3.1", + "nx": "^22.7.7", + "sigstore": "^5.0.0", + "@sigstore/core": "^4.0.1", + "@sigstore/verify": "^4.1.2", + "@babel/plugin-transform-modules-systemjs": "^7.29.8", + "fast-uri": "^4.1.2", + "websocket-driver": "^0.7.5" } } diff --git a/frontend/packages/manager/demo/webpack.splunkapp.config.js b/frontend/packages/manager/demo/webpack.splunkapp.config.js index b8e7c82..abca922 100644 --- a/frontend/packages/manager/demo/webpack.splunkapp.config.js +++ b/frontend/packages/manager/demo/webpack.splunkapp.config.js @@ -9,4 +9,5 @@ module.exports = webpackMerge(baseConfig, { filename: 'demo.js', }, devtool: 'eval-source-map', + performance: { hints: false }, }); diff --git a/frontend/packages/manager/demo/webpack.standalone.config.js b/frontend/packages/manager/demo/webpack.standalone.config.js index fd137f7..69022e6 100644 --- a/frontend/packages/manager/demo/webpack.standalone.config.js +++ b/frontend/packages/manager/demo/webpack.standalone.config.js @@ -14,11 +14,6 @@ module.exports = webpackMerge(baseConfig, { "querystring": require.resolve("querystring-es3") } }, - module: { - rules: [ - { test: /\.(png|woff|woff2|eot|ttf|svg|otf)$/, use: {loader: 'file-loader',}} - ] - }, plugins: [ new HtmlWebpackPlugin({ hash: true, @@ -30,5 +25,6 @@ module.exports = webpackMerge(baseConfig, { } }) ], - devtool: 'eval-source-map' + devtool: 'eval-source-map', + performance: { hints: false } }); diff --git a/frontend/packages/manager/package.json b/frontend/packages/manager/package.json index 2bdab14..77ee65f 100644 --- a/frontend/packages/manager/package.json +++ b/frontend/packages/manager/package.json @@ -1,6 +1,6 @@ { "name": "@splunk/manager", - "version": "1.2.2", + "version": "1.2.3-beta.8", "license": "UNLICENSED", "scripts": { "build": "NODE_ENV=production webpack --bail --config demo/webpack.standalone.config.js", @@ -20,54 +20,54 @@ }, "main": "Manager.js", "dependencies": { - "@splunk/react-ui": "^4.25.0", + "@splunk/react-ui": "^4.47.1", "@splunk/themes": "^0.11.0", "axios": "^1.16.0", - "css-loader": "^6.7.1", + "css-loader": "^7.1.4", "history": "5.3.0", - "qs": "6.15.2", + "qs": "6.15.3", "scriptjs": "^2.5.9", - "style-loader": "^3.3.1" + "style-loader": "^4.0.0" }, "devDependencies": { "@babel/core": "^7.2.0", - "@jest/globals": "^30.3.0", + "@jest/globals": "^30.4.1", "@splunk/babel-preset": "^4.0.0", "@splunk/eslint-config": "^4.0.0", "@splunk/splunk-utils": "^2.3.4", - "@splunk/stylelint-config": "^5.0.0", + "@splunk/stylelint-config": "^5.1.0", "@splunk/webpack-configs": "^5.0.0", - "@testing-library/dom": "9.3.1", - "@testing-library/jest-dom": "^6.6.3", + "@testing-library/dom": "10.4.1", + "@testing-library/jest-dom": "^6.10.0", "@testing-library/react": "12.1.2", - "@webpack-cli/serve": "^2.0.0", + "@webpack-cli/serve": "^3.0.1", "babel-eslint": "^10.1.0", - "babel-loader": "^8.0.4", - "chai": "^3.5.0", - "css-loader": "^6.7.1", + "babel-loader": "^10.1.1", + "chai": "^6.2.2", + "css-loader": "^7.1.4", "enzyme": "^3.11.0", "enzyme-adapter-react-16": "^1.15.8", "eslint": "^7.14.0", "eslint-config-airbnb": "^18.2.1", "eslint-config-prettier": "^6.15.0", "eslint-import-resolver-webpack": "^0.13.0", - "eslint-plugin-import": "^2.22.1", - "eslint-plugin-jsx-a11y": "^6.4.1", - "eslint-plugin-react": "^7.21.5", + "eslint-plugin-import": "^2.32.0", + "eslint-plugin-jsx-a11y": "^6.10.2", + "eslint-plugin-react": "^7.37.5", "eslint-plugin-react-hooks": "^4.2.0", - "html-webpack-plugin": "^4.0.0", - "jest": "^30.3.0", - "jest-environment-jsdom": "^30.3.0", + "html-webpack-plugin": "^5.6.8", + "jest": "^30.4.2", + "jest-environment-jsdom": "^30.4.1", "jest-junit": "^17.0.0", "querystring-es3": "^0.2.1", "react": "^16.12.0", "react-dom": "^16.12.0", "react-test-renderer": "^16.12.0", "styled-components": "5.1.1", - "stylelint": "^15.11.0", - "webpack": "^5.0.0", - "webpack-cli": "^5.0.0", - "webpack-dev-server": "^5.2.3", + "stylelint": "^17.14.1", + "webpack": "^5.109.2", + "webpack-cli": "^7.2.2", + "webpack-dev-server": "^6.0.0", "webpack-merge": "^4.1.3" }, "peerDependencies": { diff --git a/frontend/packages/manager/src/ManagerStyles.js b/frontend/packages/manager/src/ManagerStyles.js index 5fc572c..ff0a10d 100644 --- a/frontend/packages/manager/src/ManagerStyles.js +++ b/frontend/packages/manager/src/ManagerStyles.js @@ -18,6 +18,4 @@ const StyledGreeting = styled.div` font-size: ${variables.fontSizeXXLarge}; `; -const - export { StyledContainer, StyledGreeting }; diff --git a/frontend/packages/manager/src/components/groups/AddDeviceModal.jsx b/frontend/packages/manager/src/components/groups/AddDeviceModal.jsx index 3bc6b71..a81f92c 100644 --- a/frontend/packages/manager/src/components/groups/AddDeviceModal.jsx +++ b/frontend/packages/manager/src/components/groups/AddDeviceModal.jsx @@ -1,11 +1,11 @@ -import React, {useCallback, useContext, useState} from 'react'; +import React, {useCallback, useContext} from 'react'; import Button from '@splunk/react-ui/Button'; import Modal from '@splunk/react-ui/Modal'; import Select from '@splunk/react-ui/Select'; import Text from '@splunk/react-ui/Text'; -import api from "../../api"; import { createDOMID } from '@splunk/ui-utils/id'; import P from '@splunk/react-ui/Paragraph'; +import api from "../../api"; import GroupContext from "../../store/group-contxt"; import validateInventoryAndGroup from "../validation/ValidateInventoryAndGroup"; import InventoryDevicesValidationContxt from "../../store/inventory-devices-validation-contxt"; @@ -22,31 +22,31 @@ function AddDeviceModal(){ const handleChangeAddress = useCallback((e, { value: val }) => { GrCtx.setAddress(val); - }, [GrCtx.setAddress]); + }, [GrCtx]); const handleChangePort = useCallback((e, { value: val }) => { GrCtx.setPort(val); - }, [GrCtx.setPort]); + }, [GrCtx]); const handleChangeVersion = useCallback((e, { value: val }) => { GrCtx.setVersion(val); - }, [GrCtx.setVersion]); + }, [GrCtx]); const handleChangeCommunity = useCallback((e, { value: val }) => { GrCtx.setCommunity(val); - }, [GrCtx.setCommunity]); + }, [GrCtx]); const handleChangeSecret = useCallback((e, { value: val }) => { GrCtx.setSecret(val); - }, [GrCtx.setSecret]); + }, [GrCtx]); const handleChangeSecurityEngine = useCallback((e, { value: val }) => { GrCtx.setSecurityEngine(val); - }, [GrCtx.setSecurityEngine]); + }, [GrCtx]); - const postDevice = (deviceObj) => { + const postDevice = useCallback((deviceObj) => { api.post("/devices/add", deviceObj) - .then((response) => { + .then(() => { GrCtx.setEditedGroupId(GrCtx.groupId); GrCtx.makeGroupsChange(); }) @@ -55,11 +55,11 @@ function AddDeviceModal(){ ErrCtx.setErrorType("error"); ErrCtx.setMessage(error.response.data.message); }) - }; + }, [GrCtx, ErrCtx]); - const updateDevice = (deviceObj, deviceId) => { + const updateDevice = useCallback((deviceObj, deviceId) => { api.post(`/devices/update/${deviceId}`, deviceObj) - .then((response) => { + .then(() => { GrCtx.setEditedGroupId(GrCtx.groupId); GrCtx.makeGroupsChange(); }) @@ -68,7 +68,7 @@ function AddDeviceModal(){ ErrCtx.setErrorType("error"); ErrCtx.setMessage(error.response.data.message); }) - }; + }, [GrCtx, ErrCtx]); const handleRequestClose = () => { ValCtx.resetAllErrors(); @@ -76,7 +76,7 @@ function AddDeviceModal(){ GrCtx.setAddDeviceOpen(false); } - const handleApply = useCallback((e) => { + const handleApply = useCallback(() => { const deviceObj = { address: GrCtx.address, port: GrCtx.port, @@ -103,17 +103,16 @@ function AddDeviceModal(){ }else{ // form is invalid const errors = validation[1]; - for (const property in errors) { + Object.keys(errors).forEach((property) => { if (errors[property].length > 0){ ValCtx.setErrors(property, errors[property]); - }else { + }else{ ValCtx.resetErrors(property); - }; - }; + } + }); }; }, - [GrCtx.address, GrCtx.port, GrCtx.version, GrCtx.community, GrCtx.secret, GrCtx.securityEngine, GrCtx.isEdit, - GrCtx.deviceId, GrCtx.setAddDeviceOpen, GrCtx.groupId] + [GrCtx, ValCtx, postDevice, updateDevice] ); return ( diff --git a/frontend/packages/manager/src/components/groups/BulkAddDeviceModal.jsx b/frontend/packages/manager/src/components/groups/BulkAddDeviceModal.jsx new file mode 100644 index 0000000..3a0690f --- /dev/null +++ b/frontend/packages/manager/src/components/groups/BulkAddDeviceModal.jsx @@ -0,0 +1,295 @@ +import React, {useCallback, useContext, useEffect, useState} from 'react'; +import Button from '@splunk/react-ui/Button'; +import Modal from '@splunk/react-ui/Modal'; +import Select from '@splunk/react-ui/Select'; +import Text from '@splunk/react-ui/Text'; +import TextArea from '@splunk/react-ui/TextArea'; +import RadioBar from '@splunk/react-ui/RadioBar'; +import FormRows from '@splunk/react-ui/FormRows'; +import { createDOMID } from '@splunk/ui-utils/id'; +import P from '@splunk/react-ui/Paragraph'; +import Message from '@splunk/react-ui/Message'; +import api from "../../api"; +import GroupContext from "../../store/group-contxt"; +import validateInventoryAndGroup from "../validation/ValidateInventoryAndGroup"; +import { validationMessage } from "../../styles/ValidationStyles"; +import { StyledModalBody, StyledModalHeader } from "../../styles/inventory/InventoryStyle"; +import { StyledModeSwitch, sectionTitle } from "../../styles/groups/GroupsStyle"; +import ErrorsModalContext from "../../store/errors-modal-contxt"; +import ValidationGroup from "../validation/ValidationGroup"; + + +const emptyRow = () => ({ + keyID: createDOMID(), + address: '', + port: '', + version: '', + community: '', + secret: '', + securityEngine: '', + errors: {}, + status: null, + message: null, +}); + +const emptySharedConfig = () => ({ + port: '', + version: '', + community: '', + secret: '', + securityEngine: '', +}); + +const isBlankRow = (row) => !row.address && !row.port && !row.version + && !row.community && !row.secret && !row.securityEngine; + +export function parseAddressList(text){ + return (text || '').split(/[\n,]+/); +} + +// Turns raw address candidates into complete six-key grid rows, applying the shared SNMP +// config to every address. +export function expandAddresses(addresses, sharedConfig){ + const seen = new Set(); + const rows = []; + addresses.forEach((raw) => { + const address = (raw || '').trim(); + if (!address || address.startsWith('#') || seen.has(address)){ + return; + } + seen.add(address); + rows.push({ + keyID: createDOMID(), + address, + port: sharedConfig.port, + version: sharedConfig.version, + community: sharedConfig.community, + secret: sharedConfig.secret, + securityEngine: sharedConfig.securityEngine, + errors: {}, + status: null, + message: null, + }); + }); + return rows; +} + +function BulkAddDeviceModal(){ + const GrCtx = useContext(GroupContext); + const ErrCtx = useContext(ErrorsModalContext); + const [rows, setRows] = useState([emptyRow()]); + const [mode, setMode] = useState('manual'); + const [pasteText, setPasteText] = useState(''); + const [sharedConfig, setSharedConfig] = useState(emptySharedConfig()); + + useEffect(() => { + if (GrCtx.bulkAddOpen){ + setRows([emptyRow()]); + setMode('manual'); + setPasteText(''); + setSharedConfig(emptySharedConfig()); + } + }, [GrCtx.bulkAddOpen]); + + const handleRequestClose = useCallback(() => { + setRows([emptyRow()]); + setMode('manual'); + setPasteText(''); + setSharedConfig(emptySharedConfig()); + GrCtx.setBulkAddOpen(false); + }, [GrCtx]); + + const handleRequestAdd = useCallback(() => { + setRows((prev) => [...prev, emptyRow()]); + }, []); + + const handleRequestRemove = useCallback((e, { index }) => { + setRows((prev) => prev.filter((_, i) => i !== index)); + }, []); + + const handleRowFieldChange = useCallback((index, field, value) => { + setRows((prev) => { + const next = [...prev]; + next[index] = { ...next[index], [field]: value }; + return next; + }); + }, []); + + const handleSharedConfigChange = useCallback((field, value) => { + setSharedConfig((prev) => ({ ...prev, [field]: value })); + }, []); + + const handleModeChange = useCallback((e, { value }) => { + setMode(value); + setRows((prev) => { + if (value === 'paste'){ + return prev.filter((row) => !isBlankRow(row)); + } + return prev.length === 0 ? [emptyRow()] : prev; + }); + }, []); + + const handleExpandAddresses = useCallback(() => { + const expanded = expandAddresses(parseAddressList(pasteText), sharedConfig); + if (expanded.length === 0){ + return; + } + setRows((prev) => [...prev.filter((row) => !isBlankRow(row)), ...expanded]); + setPasteText(''); + }, [pasteText, sharedConfig]); + + const handleApply = useCallback(() => { + const validated = rows.map((row) => { + const validationObj = { + address: row.address, + port: row.port, + version: row.version, + community: row.community, + secret: row.secret, + securityEngine: row.securityEngine, + inGroupConfig: true, + }; + const [isValid, errors] = validateInventoryAndGroup(validationObj); + return { ...row, errors, status: isValid ? null : 'invalid', message: null }; + }); + + const toSubmit = []; + validated.forEach((row, index) => { + if (row.status !== 'invalid'){ + toSubmit.push({ row, index }); + } + }); + + if (toSubmit.length === 0){ + setRows(validated); + return; + } + + api.post("/devices/add/bulk", { + groupId: GrCtx.groupId, + devices: toSubmit.map(({ row }) => ({ + address: row.address, + port: row.port, + version: row.version, + community: row.community, + secret: row.secret, + securityEngine: row.securityEngine, + })), + }).then((response) => { + const { added, results } = response.data; + results.forEach((result, i) => { + const target = validated[toSubmit[i].index]; + target.status = result.added ? 'saved' : 'failed'; + target.message = result.message; + }); + if (added > 0){ + GrCtx.setEditedGroupId(GrCtx.groupId); + GrCtx.makeGroupsChange(); + } + const remaining = validated.filter((row) => row.status !== 'saved'); + if (remaining.length === 0){ + handleRequestClose(); + }else{ + setRows(remaining); + } + }).catch((error) => { + setRows(validated); + ErrCtx.setOpen(true); + ErrCtx.setErrorType("error"); + ErrCtx.setMessage(error.response.data.message); + }); + }, [rows, GrCtx, ErrCtx, handleRequestClose]); + + return ( +
+ + + + {} +

Mode

+ + + + + {mode === 'paste' ? +
+

Addresses

+