Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions local_hooks/app_tests/code_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,10 @@ def check_light_and_dark_theme_logos(self):
logos = {"Light": app_json.get("logo"), "Dark": app_json.get("logo_dark")}

for logo_theme, logo_name in logos.items():
logo_path = os.path.join(self._app_code_dir, logo_name)
if self._parser.uv_lock_filepath:
logo_path = self._parser.uv_lock_filepath.parent / logo_name
else:
logo_path = os.path.join(self._app_code_dir, logo_name)
# Make sure logo exists before checking other things
if not logo_name:
# this will be throw as an error in json_tests
Expand Down Expand Up @@ -364,13 +367,18 @@ def _get_from_svg_tree(self, tree, prop):
return val
return None

def _get_src_dir(self):
main_module_path = self._parser.sdk_app_json.get("main_module").split(":")[0].split(".")
main_module_path.pop()
return self._parser.uv_lock_filepath.parent / "/".join(main_module_path)

@TestSuite.test
def check_python_package(self):
"""
Checks for an __init__.py existing in app directory toplevel
"""
msg = TEST_PASS_MESSAGE
app_dir = Path(self._app_code_dir)
app_dir = self._get_src_dir() if self._parser.uv_lock_filepath else Path(self._app_code_dir)
if not (init_py := app_dir / "__init__.py").is_file():
msg = "App repo does not have an `__init__.py` at toplevel. Not a python module. Adding to app directory."
init_py.touch()
Expand Down
25 changes: 25 additions & 0 deletions local_hooks/app_tests/json_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
from packaging.version import Version
from pathlib import Path

SKIP_SDK_APP = {"success": True, "message": "TEST SKIPPED - SDK app detected"}


class JSONTests(TestSuite):
def __init__(self, repo_location: Path) -> None:
Expand Down Expand Up @@ -58,6 +60,9 @@ def validate_json_schema(self):
"""
Validates the structure of the app json
"""
if self._parser.uv_lock_filepath:
return SKIP_SDK_APP

APP_TESTS_DIR = Path(__file__).parent.resolve()
schema_path = APP_TESTS_DIR / "app_schema.json"
verbose = []
Expand Down Expand Up @@ -88,6 +93,8 @@ def check_extra_fields_app_json(self):
"""
Makes sure certain fields are not in the app json
"""
if self._parser.uv_lock_filepath:
return SKIP_SDK_APP

to_remove = []
msg = TEST_PASS_MESSAGE
Expand All @@ -112,6 +119,9 @@ def check_sequential_order(self):
are sequential and zero-indexed
"""

if self._parser.uv_lock_filepath:
return SKIP_SDK_APP

def check_sequence(order_tuples):
try:
order_tuples = ((a, int(b)) for a, b in order_tuples)
Expand Down Expand Up @@ -176,6 +186,9 @@ def check_main_module(self):
"""
Verifies that the main module field of the json is a valid connector filename
"""
if self._parser.uv_lock_filepath:
return SKIP_SDK_APP

app_json = self._app_json
verbose = []
main_module = app_json["main_module"]
Expand Down Expand Up @@ -307,6 +320,9 @@ def check_action_param_prefixes(self):
"""
Every parameter has an action_result.parameter associated with it
"""
if self._parser.uv_lock_filepath:
return SKIP_SDK_APP

app_json_actions = self._app_json.get("actions")

message = TEST_PASS_MESSAGE
Expand Down Expand Up @@ -358,6 +374,9 @@ def check_action_param_matching_contains(self):
"""
Every parameter for an action with contains has an action_result.parameter with the same contains
"""
if self._parser.uv_lock_filepath:
return SKIP_SDK_APP

action_list = [act for act in self._app_json["actions"]]
verbose = []
for index, action in enumerate(action_list):
Expand Down Expand Up @@ -399,6 +418,9 @@ def check_minimal_data_paths(self):
"""
Checks to make sure each action includes the minimal required data paths
"""
if self._parser.uv_lock_filepath:
return SKIP_SDK_APP

app_json_actions = [act for act in self._app_json["actions"]]
verbose = []
message = TEST_PASS_MESSAGE
Expand Down Expand Up @@ -508,6 +530,9 @@ def check_pip313_dependencies(self):
"""
Ensures pip313_dependencies key exists and has appropriate content based on requirements.txt
"""
if self._parser.uv_lock_filepath:
return SKIP_SDK_APP

verbose = []
message = TEST_PASS_MESSAGE

Expand Down
26 changes: 26 additions & 0 deletions local_hooks/app_tests/utils/app_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,12 @@
from .phantom_constants import APP_EXTS
from functools import cached_property
from pathlib import Path
from typing import Optional
from local_hooks.helpers import (
find_uv_lock_file,
load_sdk_apps_enviornment,
generate_sdk_app_manifest,
)


class ParserError(Exception):
Expand Down Expand Up @@ -104,10 +110,22 @@ def find_app_json_name(self, json_filenames):
def _app_json_filepath(self) -> Path:
return self._app_code_dir / self.app_json_name

@cached_property
def sdk_app_json(self) -> Path:
uv_lock_dir = self.uv_lock_filepath.parent
load_sdk_apps_enviornment(uv_lock_dir)
return generate_sdk_app_manifest(uv_lock_dir)

@cached_property
def uv_lock_filepath(self) -> Optional[Path]:
return find_uv_lock_file(self._app_code_dir)

@cached_property
def app_json(self):
# Gets the loaded json, preserving key order
try:
if self.uv_lock_filepath:
return self.sdk_app_json
json_content = json.loads(self._app_json_filepath.read_text())
return json_content
except FileNotFoundError as e:
Expand Down Expand Up @@ -143,6 +161,14 @@ def connector_filepath(self):
)
) from None

if not connector_filename.endswith(".py"):
# sdkfied app
if self.uv_lock_filepath:
path_to_main_module = connector_filename.split(":")[0]
full_path = "/".join(path_to_main_module.split("."))
full_path += ".py"
return self.uv_lock_filepath.parent / full_path

if connector_filename in self.filenames:
return os.path.join(self._app_code_dir, connector_filename)

Expand Down
19 changes: 19 additions & 0 deletions local_hooks/build_docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@
from jinja2 import Environment, FileSystemLoader, select_autoescape
from jinja2.ext import Extension
from jinja2.lexer import Token, TokenStream
from local_hooks.helpers import (
find_uv_lock_file,
load_sdk_apps_enviornment,
generate_sdk_app_manifest,
)

SCRIPT_DIR = Path(__file__).parent.resolve()
TEMPLATE_DIR = SCRIPT_DIR / "templates"
Expand Down Expand Up @@ -88,6 +93,20 @@ def escape_markdown(data: Any) -> Any:

def get_app_json(app_json_dir, json_name):
logging.info("Looking for app JSON in: %s", app_json_dir)

# Check if this is an SDK app and generate manifest if needed
if uv_lock_path := find_uv_lock_file(app_json_dir):
uv_lock_dir = uv_lock_path.parent
logging.info("SDK app detected, generating manifest: UV_LOCK_DIR=%s", uv_lock_dir)

# Install dependencies from pyproject.toml using uv
load_sdk_apps_enviornment(uv_lock_dir)

# Generate the SDK manifest in a temporary directory
logging.info("Generating SDK app manifest in temporary directory")
return generate_sdk_app_manifest(uv_lock_dir)

# Handle non-SDK apps - file-based JSON loading
if not json_name:
try:
app_json_name = next(
Expand Down
32 changes: 29 additions & 3 deletions local_hooks/generate_notice.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
import dataclasses
import logging
from pathlib import Path
import toml
from local_hooks.helpers import find_uv_lock_file


# pip-licenses is used to query all python packages for license information
PYTHON_LICENSE_COMMAND = "pip-licenses"
Expand Down Expand Up @@ -162,10 +165,18 @@ def get_python_license_info(packages: list[str]):
yield LicenseLine.make_from_pip_json(license_info)


def get_app_json(connector_path: Path) -> tuple[str, str]:
def get_app_json(connector_path: Path, uv_lock_path: Optional[Path]) -> tuple[str, str]:
"""
Get the app name and license from the app.json file.
"""

if uv_lock_path:
with open(uv_lock_path / "pyproject.toml") as f:
toml_data = toml.load(f)
app_name = toml_data.get("project", {}).get("name")
app_license = toml_data.get("project", {}).get("license")
return app_name, app_license

logging.info("Looking for app JSON in: %s", connector_path)
json_files = glob.glob(os.path.join(connector_path, "*.json"))
# Exclude files with the pattern '*.postman_collection.json'
Expand Down Expand Up @@ -224,6 +235,16 @@ def remove_trailing_blank_lines(notice_file_path: Path):
f.write("\n")


def get_sdkfied_app_dependencies(pyproject_toml_path: Path) -> list[str]:
"""
Get the dependencies from the pyproject.toml file.
"""
with open(pyproject_toml_path) as f:
toml_data = toml.load(f)

return toml_data.get("project", {}).get("dependencies", [])


def main():
"""
Generate a NOTICE file.
Expand All @@ -234,7 +255,8 @@ def main():
args = parse_args()
connector_path = Path(args.connector_path)

app_name, app_license = get_app_json(connector_path)
uv_lock_path = find_uv_lock_file(connector_path)
app_name, app_license = get_app_json(connector_path, uv_lock_path.parent)
notice_file_path = connector_path / "NOTICE"
logging.info("Creating NOTICE file at %s", Path(notice_file_path.resolve()))

Expand All @@ -243,7 +265,11 @@ def main():
f.write(f"Splunk SOAR App: {app_name}\n{app_license}\n")

# Get all python package dependencies
packages = get_package_dependencies()
if uv_lock_path:
uv_lock_dir = uv_lock_path.parent
packages = get_sdkfied_app_dependencies(uv_lock_dir / "pyproject.toml")
else:
packages = get_package_dependencies()
valid_packages = [
package for package in packages if package not in EXCLUDED_PYTHON_PACKAGES
]
Expand Down
110 changes: 110 additions & 0 deletions local_hooks/helpers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
import logging
from pathlib import Path
from typing import Optional
import subprocess
import tempfile
import json
import shutil
import os


def ensure_uv_available():
Comment thread
tapishj-splunk marked this conversation as resolved.
Resolved
"""
Check if uv is installed and install it if not.
Mimics the GitHub pipeline approach:
curl -LsSf https://astral.sh/uv/install.sh | sh
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
"""
# Check if uv is already available
if shutil.which("uv"):
logging.info("uv is already installed and available")
return

logging.info("uv not found, installing")

try:
# Download and execute the uv installer (equivalent to: curl -LsSf https://astral.sh/uv/install.sh | sh)
logging.info("Downloading uv installer...")
result = subprocess.run(
["curl", "-LsSf", "https://astral.sh/uv/install.sh"],
capture_output=True,
text=True,
check=True,
)

logging.info("Executing uv installer...")
subprocess.run(["sh"], input=result.stdout, text=True, check=True)

# Add ~/.cargo/bin to PATH for current session (equivalent to: echo "$HOME/.cargo/bin" >> $GITHUB_PATH)
cargo_bin = os.path.expanduser("~/.cargo/bin")
current_path = os.environ.get("PATH", "")
if cargo_bin not in current_path:
os.environ["PATH"] = f"{cargo_bin}:{current_path}"
logging.info("Added %s to PATH", cargo_bin)

except subprocess.CalledProcessError as e:
logging.error("Failed to install uv: %s", e)
raise RuntimeError(f"uv installation failed: {e}")


def find_uv_lock_file(connector_path: Path) -> Optional[Path]:
"""
Find uv.lock file in the connector_path or its subdirectories.
Returns the path to the uv.lock file if found, None otherwise.
"""
# By convention, SDK apps are managed by uv and old apps are not
for uv_lock_path in connector_path.rglob("uv.lock"):
logging.info("Found uv.lock at: %s", uv_lock_path)
return uv_lock_path


def load_sdk_apps_enviornment(uv_lock_dir: Path):
"""
Load the SDK app's environment.
"""
ensure_uv_available()

pyproject_path = uv_lock_dir / "pyproject.toml"
if pyproject_path.exists():
logging.info("Installing dependencies from pyproject.toml")
try:
subprocess.run(
["uv", "sync"], cwd=uv_lock_dir, check=True, capture_output=True, text=True
)
logging.info("Successfully installed dependencies")

except subprocess.CalledProcessError as e:
logging.error("Failed to install dependencies: %s", e)
raise e
else:
raise ValueError("No pyproject.toml found in %s", uv_lock_dir)


def generate_sdk_app_manifest(uv_lock_dir: Path) -> dict:
ensure_uv_available()

with tempfile.TemporaryDirectory() as temp_dir:
temp_manifest_path = Path(temp_dir) / "sdk_app_manifest.json"
try:
# Use uv run to execute soarapps in the SDK app's environment
subprocess.run(
[
"uv",
"run",
"soarapps",
"manifests",
"create",
str(temp_manifest_path),
".",
],
check=True,
cwd=uv_lock_dir, # Run from the SDK app directory
)
logging.info("Successfully generated SDK manifest in temp directory")

with open(temp_manifest_path) as f:
return json.load(f)

except subprocess.CalledProcessError as e:
logging.error("Failed to generate SDK manifest: %s", e)
raise e
13 changes: 13 additions & 0 deletions local_hooks/tests/data/static_tests/sdk_app_dir/pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
[project]
name = "test"
version = "4.0.0"
description = "This is a test app"
license = "Copyright (c) 2025 Splunk Inc."
requires-python = ">=3.9, <3.14"
authors = [
]
dependencies = [
"splunk-soar-sdk==1.6.0",
"validators",
"xmltodict",
]
Loading