diff --git a/Dockerfile b/Dockerfile index c6216aa69a..452cf37e66 100644 --- a/Dockerfile +++ b/Dockerfile @@ -145,6 +145,7 @@ RUN apt-get update \ libguestfs-tools \ linux-image-amd64 \ openjdk-17-jre-headless \ + docker.io \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/docker-compose.yml b/docker-compose.yml index 9da2c248eb..5a3bf408e5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,16 @@ services: timeout: 5s retries: 5 + # Add the Docker-in-Docker daemon + dind: + image: docker.io/library/docker:dind + privileged: true + environment: + - DOCKER_TLS_CERTDIR= + volumes: + - dind_data:/var/lib/docker + - workspace:/var/scancodeio/workspace/ + web: build: . command: sh -c " @@ -38,6 +48,8 @@ services: ./manage.py collectstatic --no-input --verbosity 0 --clear && gunicorn scancodeio.wsgi:application --bind :8000 --timeout 600 \ --workers ${GUNICORN_WORKERS:-8} --worker-tmp-dir /dev/shm" + environment: + - DOCKER_HOST=tcp://dind:2375 # Point to the DinD container env_file: - docker.env expose: @@ -61,6 +73,8 @@ services: ./manage.py rqworker --worker-class scancodeio.worker.ScanCodeIOWorker --queue-class scancodeio.worker.ScanCodeIOQueue --verbosity 1" + environment: + - DOCKER_HOST=tcp://dind:2375 # Point to the DinD container env_file: - docker.env volumes: @@ -75,6 +89,8 @@ services: condition: service_healthy web: condition: service_started + dind: + condition: service_started nginx: image: docker.io/library/nginx:1.31.4-alpine @@ -104,3 +120,4 @@ volumes: static: workspace: webroot: + dind_data: diff --git a/docs/built-in-pipelines.rst b/docs/built-in-pipelines.rst index 644f81ee03..4368ab4eb3 100644 --- a/docs/built-in-pipelines.rst +++ b/docs/built-in-pipelines.rst @@ -281,6 +281,12 @@ Scan Maven Package :members: :member-order: bysource +Scan Nix Package +------------------- +.. autoclass:: scanpipe.pipelines.scan_nix_package.ScanNixPackage() + :members: + :member-order: bysource + Fetch Scores (addon) -------------------- .. warning:: diff --git a/pyproject.toml b/pyproject.toml index 6256fa28f3..f8ba8bb755 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -175,6 +175,7 @@ resolve_dependencies = "scanpipe.pipelines.resolve_dependencies:ResolveDependenc scan_codebase = "scanpipe.pipelines.scan_codebase:ScanCodebase" scan_for_virus = "scanpipe.pipelines.scan_for_virus:ScanForVirus" scan_maven_package = "scanpipe.pipelines.scan_maven_package:ScanMavenPackage" +scan_nix_package = "scanpipe.pipelines.scan_nix_package:ScanNixPackage" scan_single_package = "scanpipe.pipelines.scan_single_package:ScanSinglePackage" [tool.setuptools.packages.find] diff --git a/scanpipe/pipelines/scan_nix_package.py b/scanpipe/pipelines/scan_nix_package.py new file mode 100644 index 0000000000..b2cb424f28 --- /dev/null +++ b/scanpipe/pipelines/scan_nix_package.py @@ -0,0 +1,239 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import shutil +from pathlib import Path + +from scanpipe.pipelines.deploy_to_develop import DeployToDevelop +from scanpipe.pipelines.scan_codebase import ScanCodebase +from scanpipe.pipelines.scan_single_package import ScanSinglePackage +from scanpipe.pipes import d2d +from scanpipe.pipes import flag +from scanpipe.pipes import nix +from scanpipe.pipes import utils +from scanpipe.pipes.nix import check_input_and_return_purl +from scanpipe.pipes.nix import cleanup_docker_volumes +from scanpipe.pipes.nix import fetch_inputs + + +class ScanNixPackage(ScanSinglePackage, DeployToDevelop, ScanCodebase): + """ + Download the nix source and binary, and run a deployment to development + scan between the binary and the source to detect any discrepancies. + + Scan the sources and confirm that the detected license aligns with + the declared license that is detected from the nix package. + """ + + download_inputs = False + + @classmethod + def steps(cls): + return ( + cls.check_input_and_return_purl, + cls.check_docker_command, + cls.fetch_inputs, + cls.collect_input_info, + cls.extract_input_to_codebase_directory, + cls.collect_and_create_codebase_resources, + cls.scan_for_application_packages, + cls.scan_for_files, + cls.clear_to_codebase_status, + cls.collect_and_create_license_detections, + cls.add_from_to_tag, + cls.d2d_steps, + cls.validate_package_license_integrity, + cls.cleanup_docker_volumes, + ) + + def check_input_and_return_purl(self): + """Validate the input is a PURL string and return the PURL object.""" + self.purl = check_input_and_return_purl(self.project) + + def check_docker_command(self): + """Check if the Docker command is available.""" + if not utils.check_docker_command(): + raise Exception("Docker is required and its daemon must be running.") + nix.ensure_multiarch_emulation() + + def fetch_inputs(self): + """Fetch the binary and source of the given PURL.""" + from_file = "" + to_file = "" + output_format = "" + from_file, to_file, output_format, error_messages, warning_messages = ( + fetch_inputs(self.purl, self.project.codebase_path) + ) + self.from_file = from_file + self.to_file = to_file + self.output_format = output_format + + self.d2d_enable = bool(self.from_file and self.to_file) + + if error_messages: + self.project.add_error(error_messages) + if warning_messages: + self.project.add_warning(warning_messages) + + def collect_input_info(self): + """Collect information about the input.""" + self.input_path = "" + if self.to_file: + self.input_path = Path(self.to_file) + self.collect_input_information() + + def extract_input_to_codebase_directory(self): + """Extract input to project codebase/ directory.""" + if self.input_path: + extracted_path = nix.extract_nar_archive( + self.input_path, self.project.codebase_path, self.output_format + ) + + to_dir = Path(self.project.codebase_path) / "to" + # If the extraction failed (returned "") or we found it was empty + if not extracted_path or (to_dir.exists() and not list(to_dir.rglob("*"))): + if to_dir.exists(): + shutil.rmtree(to_dir) + self.d2d_enable = False + self.project.add_error( + "Failed to extract NAR archive, D2D scan disabled." + ) + + self.env = self.project.get_env() + + def clear_to_codebase_status(self): + """ + Clear the status of the to codebase resources in the project as + having status will prevent D2D from running. + """ + flag.clear_status(self.project.codebaseresources.to_codebase()) + + def add_from_to_tag(self): + """Update 'from' and 'to' tag to resources based on their path.""" + if self.d2d_enable: + d2d.update_from_to_tag(self.project) + + def d2d_steps(self): + """ + Run the deployment to development scan if both the source and + binary are available. + """ + if self.d2d_enable: + self.flag_empty_files() + self.flag_whitespace_files() + self.flag_ignored_resources() + self.map_about_files() + self.map_checksum() + self.match_archives_to_purldb() + self.load_ecosystem_config() + self.d2d_java() + self.d2d_scala() + self.d2d_kotlin() + self.d2d_grammar() + self.d2d_groovy() + self.d2d_aspectj() + self.d2d_clojure() + self.d2d_xtend() + self.d2d_javascript() + self.d2d_process() + + def d2d_java(self): + self.find_java_packages() + self.map_java_to_class() + self.map_jar_to_java_source() + + def d2d_scala(self): + self.find_scala_packages() + self.map_scala_to_class() + self.map_jar_to_scala_source() + + def d2d_kotlin(self): + self.find_kotlin_packages() + self.map_kotlin_to_class() + self.map_jar_to_kotlin_source() + + def d2d_grammar(self): + self.find_grammar_packages() + self.map_grammar_to_class() + self.map_jar_to_grammar_source() + + def d2d_groovy(self): + self.find_groovy_packages() + self.map_groovy_to_class() + self.map_jar_to_groovy_source() + + def d2d_aspectj(self): + self.find_aspectj_packages() + self.map_aspectj_to_class() + self.map_jar_to_aspectj_source() + + def d2d_clojure(self): + self.find_clojure_packages() + self.map_clojure_to_class() + self.map_jar_to_clojure_source() + + def d2d_xtend(self): + self.find_xtend_packages() + self.map_xtend_to_class() + + def d2d_javascript(self): + self.map_javascript() + self.map_javascript_symbols() + self.map_javascript_strings() + + def d2d_process(self): + self.get_symbols_from_binaries() + self.map_elf() + self.map_macho() + self.map_winpe() + self.map_go() + self.map_rust() + self.map_python() + self.match_directories_to_purldb() + self.match_resources_to_purldb() + self.map_javascript_post_purldb_match() + self.map_javascript_path() + self.map_javascript_colocation() + self.map_thirdparty_npm_packages() + self.map_path() + self.flag_mapped_resources_archives_and_ignored_directories() + self.perform_house_keeping_tasks() + self.match_purldb_resources_post_process() + self.remove_packages_without_resources() + self.flag_deployed_from_resources_with_missing_license() + self.create_local_files_packages() + + def validate_package_license_integrity(self): + """ + Validate the correctness of the package license compare with the + detected license from the codebase. + """ + utils.validate_package_license_integrity(self.project) + + def flag_mapped_status(self): + """Flag the from codebase resources that were mapped.""" + if self.d2d_enable: + flag.flag_mapped_resources(self.project) + + def cleanup_docker_volumes(self): + """Cleanup the Docker volumes used for Nix.""" + cleanup_docker_volumes() diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index eda7703e38..962a342bbc 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -141,10 +141,14 @@ def _map_checksum_resource(to_resource, from_resources, checksum_field): def map_checksum(project, checksum_field, logger=None): """Map using checksum.""" - project_files = project.codebaseresources.files().no_status() - from_resources = project_files.from_codebase().has_value(checksum_field) + from_resources = ( + project.codebaseresources.files().from_codebase().has_value(checksum_field) + ) to_resources = ( - project_files.to_codebase().has_value(checksum_field).has_no_relation() + project.codebaseresources.files() + .to_codebase() + .has_value(checksum_field) + .has_no_relation() ) resource_count = to_resources.count() @@ -1767,20 +1771,25 @@ def map_paths_resource( relations_to_create[rel_key] = relation if paths_not_mapped: to_resource.status = flag.REQUIRES_REVIEW - logger( - f"WARNING: #{len(paths_not_mapped)} {map_type} paths NOT mapped for: " - f"{to_resource.path!r}" - ) + if logger: + logger( + f"WARNING: #{len(paths_not_mapped)} {map_type} paths NOT " + f" mapped for: {to_resource.path!r}" + ) to_resource.save() if relations_to_create: rels = CodebaseRelation.objects.bulk_create(relations_to_create.values()) - logger( - f"Created {len(rels)} mappings using " - f"{', '.join(map_types)} for: {to_resource.path!r}" - ) + if logger: + logger( + f"Created {len(rels)} mappings using " + f"{', '.join(map_types)} for: {to_resource.path!r}" + ) else: - logger(f"No mappings using {', '.join(map_types)} for: {to_resource.path!r}") + if logger: + logger( + f"No mappings using {', '.join(map_types)} for: {to_resource.path!r}" + ) def process_paths_in_binary( @@ -1944,9 +1953,17 @@ def map_elfs_with_dwarf_paths(project, logger=None): f"with {from_resources.count():,d} from/ resources." ) - from_resources_index = pathmap.build_index( - from_resources.values_list("id", "path"), with_subpaths=True - ) + # Build the path index, adding virtual aliases for .in template files + from_paths = [] + for res_id, path in from_resources.values_list("id", "path"): + from_paths.append((res_id, path)) + # If the source file is a template ending in '.in', also index its + # target name + if path.endswith(".in"): + target_path = path[:-3] # Removes the trailing '.in' + from_paths.append((res_id, target_path)) + + from_resources_index = pathmap.build_index(from_paths, with_subpaths=True) if logger: logger("Done building from/ resources index.") @@ -2035,6 +2052,15 @@ def map_go_paths(project, logger=None): ) +def update_from_to_tag(project): + """Update 'from' or 'to' tag to resources based on their path.""" + for resource in project.codebaseresources.files(): + if resource.path.startswith("from/"): + resource.update(tag="from") + elif resource.path.startswith("to/"): + resource.update(tag="to") + + RUST_BINARY_OPTIONS = ["Rust"] ELF_BINARY_OPTIONS = ["Python", "Go", "Elf"] MACHO_BINARY_OPTIONS = ["Rust", "Go", "MacOS"] diff --git a/scanpipe/pipes/fetch.py b/scanpipe/pipes/fetch.py index 3cbbb13200..401824f76f 100644 --- a/scanpipe/pipes/fetch.py +++ b/scanpipe/pipes/fetch.py @@ -82,6 +82,13 @@ def get_request_session(uri): """Return a Requests session setup with authentication and headers.""" session = requests.Session() + + # Set a default User-Agent to avoid 403 Forbidden errors on strict + # registries that block default python-requests headers. + session.headers.update( + {"User-Agent": "ScanCode.io (https://github.com/aboutcode-org/scancode.io)"} + ) + netloc = urlparse(uri).netloc if credentials := scanpipe_settings.FETCH_BASIC_AUTH.get(netloc): diff --git a/scanpipe/pipes/flag.py b/scanpipe/pipes/flag.py index ad366045a0..087ff81935 100644 --- a/scanpipe/pipes/flag.py +++ b/scanpipe/pipes/flag.py @@ -66,6 +66,7 @@ REQUIRES_REVIEW = "requires-review" REVIEW_DANGLING_LEGAL_FILE = "review-dangling-legal-file" NOT_DEPLOYED = "not-deployed" +LICENSE_ISSUE = "license-mismatch-declared-vs-detected" GENERATED = "generated-file" @@ -138,3 +139,8 @@ def flag_mapped_resources(project): """Flag all codebase resources that were mapped during the d2d pipeline.""" resources = project.codebaseresources.has_relation().no_status() return resources.update(status=MAPPED) + + +def clear_status(resource_qs): + """Clear the status of given codebase resources.""" + return resource_qs.update(status="") diff --git a/scanpipe/pipes/nix.py b/scanpipe/pipes/nix.py new file mode 100644 index 0000000000..60bbbe4139 --- /dev/null +++ b/scanpipe/pipes/nix.py @@ -0,0 +1,559 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import atexit +import logging +import shutil +import subprocess +from pathlib import Path + +import requests +from fetchcode import fetch_json_response +from packageurl import PackageURL + +from scanpipe.pipes import utils + +logger = logging.getLogger(__name__) + + +def check_input_and_return_purl(project): + """Validate the input and return a Nix PURL.""" + input_sources = project.inputsources.all() + if len(input_sources) != 1: + error_msg = "Only 1 nix purl is accepted." + raise ValueError(error_msg) + + project_input = str(input_sources[0]) + input_purl = PackageURL.from_string(project_input) + if input_purl.type != "nix": + error_msg = "Only nix purl is supported." + raise ValueError(error_msg) + + namespace = input_purl.namespace + if not namespace or namespace.lower() != "nixpkgs": + raise Exception( + "Only official nixpkgs repository is supported (i.e. namespace=nixpkgs)." + ) + + qualifiers = input_purl.qualifiers or {} + if not input_purl.version and "commit" not in qualifiers: + raise Exception("Version or a 'commit' qualifier is required.") + + if "system" not in qualifiers: + raise Exception( + "The 'system' qualifier is required to resolve system-specific binaries." + ) + + return input_purl + + +def fetch_inputs(purl, output_dir): + """ + Fetch the system specific binary and the exact source tree with the + patches and configurations applied for the given input purl. Return a + tuple of (source_path, binary_path, output_format, error_message, + warning_message). + """ + data = get_package_data(purl) + name = purl.name + version = purl.version + + commit_hash = purl.qualifiers.get("commit", "") + system = purl.qualifiers.get("system", "") + user_output = purl.qualifiers.get("output", "") + error_message = "" + warning_message = "" + + output_format, path, release_commit_hash = get_nix_store_path( + data, name, version, system, commit_hash, user_output + ) + + concluded_commit_hash = release_commit_hash or commit_hash + + bin_path = "" + nix_bin_download_url = get_nix_download_url(path) if path else "" + # Try to download from cache first + if nix_bin_download_url: + bin_path = utils.fetch_path(nix_bin_download_url) + + if bin_path: + logger.info(f"Downloaded binary for {purl} to {bin_path}") + else: + if concluded_commit_hash: + logger.info( + f"Binary not found in cache for {purl}. Attempting local Nix build..." + ) + bin_path = build_binary_with_docker( + name, output_dir, system, concluded_commit_hash, output_format + ) + if bin_path: + logger.info(f"Successfully built binary for {purl} to {bin_path}") + warning_message = ( + f"Binary not found in cache for {purl}. Built locally using " + f"commit {concluded_commit_hash} with a Linux-based Nix " + f"Docker container." + ) + logger.warning(warning_message) + else: + error_message = f"Failed to fetch or build the binary for {purl}" + logger.error(error_message) + + src_path = "" + if concluded_commit_hash: + src_path = get_patched_source_with_docker( + name, output_dir, system, concluded_commit_hash + ) + + return src_path, bin_path, output_format, error_message, warning_message + + +def build_binary_with_docker(name, output_dir, system, commit_hash, output_format): + """ + Fetch a Nix package and build its binary from source using Docker. + Exports the resulting store path as a .nar file for standard extraction. + """ + nar_filename = f"{name}-bin.nar" + extracted_path = Path(output_dir) / nar_filename + absolute_out_dir = str(Path(output_dir).resolve()) + + # Handle architecture and system incompatibilities + target_os = system.split("-")[-1] if "-" in system else system + if target_os and target_os != "linux": + logger.warning( + f"SYSTEM BARRIER DETECTED: Target system '{system}' requires " + f"OS-specific SDKs that cannot be evaluated inside the " + f"Linux-based Nix Docker container. Defaulting the build to " + f"the container's native Linux architecture." + ) + system_config = "" + else: + system_config = ( + f'localSystem = builtins.currentSystem; crossSystem = "{system}";' + ) + + config_str = ( + "config = { " + "allowBroken = true; " + "allowUnfree = true; " + "allowUnsupportedSystem = true; " + "};" + ) + + nixpkgs_import = ( + f'import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/' + f'{commit_hash}.tar.gz") {{ {system_config} {config_str} }}' + ) + + # Defaulting to 'debug' if none is specified. + effective_output = output_format or "debug" + + # Fall back to the default target if the effective_output is not + # defined in the recipe for this package. + nix_expression = ( + f"let " + f" pkgs = {nixpkgs_import}; " + f" target = pkgs.{name}; " + f" hasIt = builtins.isAttrs target && " + f'builtins.hasAttr "{effective_output}" target; ' + f"in if hasIt then target.{effective_output} else target" + ) + + # Build the Nix package, verify it succeeded, and export the output as + # a .nar file. + container_script = f""" + OUT_PATH=$(nix-build --no-out-link -E '{nix_expression}') + if [ -z "$OUT_PATH" ] || [ ! -e "$OUT_PATH" ]; then + echo "Error: nix-build failed to return a valid store path." >&2 + exit 1 + fi + nix-store --dump "$OUT_PATH" > /build_output/{nar_filename} + """ + + cmd = [ + "docker", + "run", + "--rm", + "-v", + "nix-eval-cache:/nix", + "-v", + f"{absolute_out_dir}:/build_output", + "nixos/nix", + "/bin/sh", + "-c", + container_script, + ] + + task_description = f"Building ({name} for {system})" + + try: + subprocess.run(cmd, capture_output=True, text=True, check=True, timeout=1800) # noqa: S603 + if extracted_path.exists(): + return str(extracted_path) + return "" + except subprocess.CalledProcessError as e: + logger.error(f"Failed: {task_description} with error: {e.stderr.strip()}") + except subprocess.TimeoutExpired: + logger.error(f"Failed: {task_description} with error: Process timed out") + return "" + + +def get_nix_store_path(data, name, version, system, commit_hash, user_output): + """Get the Nix store path and release commit hash.""" + outputs_to_try = [user_output] if user_output else ["debug", "out"] + path = "" + release_commit_hash = "" + output_format = "" + + for output in outputs_to_try: + if data: + release_commit_hash, path = get_commit_hash_nix_store_path( + data, system, output, version, commit_hash + ) + + if not data or not path: + if commit_hash: + path = get_nix_store_path_with_nix(name, system, output, commit_hash) + + if path: + output_format = output + break + + if not path: + if not commit_hash: + raise Exception( + "Please provide a 'commit' qualifier in the PURL " + "for Nix to determine the download URL or build it locally." + ) + output_format = user_output or "debug" + + return output_format, path, release_commit_hash + + +def get_commit_hash_nix_store_path(data, system, output, version, commit_hash=""): + """ + Find and return the commit_hash and store path (/nix/store/) + based on the qualifiers + """ + releases = data.get("releases") or [] + releases = [r for r in releases if r.get("version") == version] + + for release in releases: + release_version = release.get("version", "") + if version and release_version != version: + continue + for platform in release.get("platforms", []): + release_commit_hash = platform.get("commit_hash", "") + if platform.get("system") != system: + continue + if commit_hash and release_commit_hash != commit_hash: + continue + for out in platform.get("outputs", []): + if out.get("name") == output: + return release_commit_hash, out.get("path") + return "", "" + + +def get_package_data(purl): + """Fetch package data from https://search.devbox.sh/.""" + api_url = f"https://search.devbox.sh/v2/pkg?name={purl.name}" + try: + return fetch_json_response(api_url) + except Exception as e: + logger.warning(f"Failed to fetch package data for {purl}: {e}") + return None + + +def get_nix_store_path_with_nix(name, system, output, commit_hash): + """Find and return the store path using 'nix'""" + system_config = f'system = "{system}";' if system else "" + config_str = "config = { allowBroken = true; allowUnfree = true; };" + + nix_expression = ( + "let " + f" pkgs = import (fetchTarball " + f'"https://github.com/NixOS/nixpkgs/archive/{commit_hash}.tar.gz") ' + f"{{ {system_config} {config_str} }}; " + f" target = pkgs.{name}; " + f' hasIt = builtins.isAttrs target && builtins.hasAttr "{output}" target; ' + f'in if hasIt then target.{output}.outPath else ""' + ) + + cmd = [ + "docker", + "run", + "--rm", + "-v", + "nix-eval-cache:/nix", + "nixos/nix", + "nix-instantiate", + "--eval", + "--raw", + "-E", + nix_expression, + ] + + try: + result = subprocess.run( # noqa: S603 + cmd, capture_output=True, text=True, check=True, timeout=300 + ) + return result.stdout.strip() + except subprocess.CalledProcessError as e: + logger.error(f"Error evaluating attribute for package '{name}': {e.stderr}") + return "" + except subprocess.TimeoutExpired: + logger.error(f"Timeout evaluating attribute for package '{name}'") + return "" + + +def get_nix_download_url(path): + """Construct a download url from cache.nixos.org based on store path""" + base_name = path.rstrip("/").split("/")[-1] + narinfo_hash = base_name.split("-")[0] + + narinfo_url = f"https://cache.nixos.org/{narinfo_hash}.narinfo" + url_path = get_narinfo_url(narinfo_url) + + if not url_path: + logger.warning(f"{narinfo_url} is not accessible.") + return "" + + return f"https://cache.nixos.org/{url_path}" + + +def get_narinfo_url(narinfo_url): + """Visit the narinfo url, parse and return the URL value""" + try: + response = requests.get(narinfo_url, timeout=10) + response.raise_for_status() + except requests.exceptions.RequestException: + return "" + + for line in response.text.splitlines(): + if line.startswith("URL:"): + return line.split(":", 1)[1].strip() + + return "" + + +def cleanup_docker_volumes(): + """Cleanup the Docker volumes used for Nix.""" + if not shutil.which("docker"): + return + + cmd = ["docker", "volume", "rm", "-f", "nix-eval-cache"] + try: + subprocess.run(cmd, capture_output=True, check=False) # noqa: S603 + except Exception as e: + logger.debug(f"Failed to cleanup Docker volumes: {e}") + + +atexit.register(cleanup_docker_volumes) + + +def extract_nar_archive(archive_path, output_dir, output): + """Extract a compressed Nix NAR archive.""" + archive_path = Path(archive_path).resolve() + output_dir = Path(output_dir).resolve() + output_dir.mkdir(parents=True, exist_ok=True) + + archive_dir = str(archive_path.parent) + archive_name = archive_path.name + + extracted_path = output_dir / "to" / output + + if archive_name.endswith(".xz"): + compression_type = "xz" + decompress_cmd = f"xzcat /input/{archive_name}" + elif archive_name.endswith(".zst"): + compression_type = "zstd" + decompress_cmd = f"zstdcat /input/{archive_name}" + elif archive_name.endswith(".bz2"): + compression_type = "bzip2" + decompress_cmd = f"bzcat /input/{archive_name}" + elif archive_name.endswith(".gz"): + compression_type = "gzip" + decompress_cmd = f"zcat /input/{archive_name}" + else: + compression_type = None + decompress_cmd = f"cat /input/{archive_name}" + + if compression_type: + restore_pipeline = ( + f"nix-shell -p {compression_type} --run " + f"'{decompress_cmd} | nix-store --restore /output/to/{output}'" + ) + else: + restore_pipeline = f"{decompress_cmd} | nix-store --restore /output/to/{output}" + + container_script = ( + f"rm -rf /output/to/{output} && mkdir -p /output/to && {restore_pipeline}" + ) + + cmd = [ + "docker", + "run", + "--rm", + "-v", + "nix-eval-cache:/nix", + "-v", + f"{archive_dir}:/input:ro", + "-v", + f"{output_dir}:/output", + "nixos/nix", + "/bin/sh", + "-c", + container_script, + ] + + try: + subprocess.run(cmd, capture_output=True, text=True, check=True, timeout=600) # noqa: S603 + return str(extracted_path) + except subprocess.CalledProcessError as e: + logger.error(f"Failed to extract {archive_name} with error: {e.stderr.strip()}") + return "" + except subprocess.TimeoutExpired: + logger.error(f"Failed to extract {archive_name}: Process timed out") + return "" + + +def get_patched_source_with_docker(name, output_dir, system, commit_hash): + """Fetch a Nix package source and apply its official patches.""" + extracted_path = Path(output_dir) / "from" + extracted_path.mkdir(parents=True, exist_ok=True) + absolute_out_dir = str(extracted_path.resolve()) + + # Get the OS part from the system string (e.g. 'aarch64-darwin' to 'darwin') + target_os = system.split("-")[-1] if "-" in system else system + + # Check for OS incompatibility + # Since the Docker container uses 'nixos/nix' which is linux-based, it + # cannot build the patched sources for other systems + if target_os and target_os != "linux": + logger.warning( + f"SYSTEM BARRIER DETECTED: Target system '{system}' requires " + f"OS-specific SDKs that cannot be evaluated inside the " + f"Linux-based Nix Docker container." + ) + logger.warning( + f"FALLBACK IN EFFECT: Evaluating the source using the container's " + f"native Linux environment. The extracted source tree will " + f"contain Linux-specific patches instead of {system} patches. " + f"Impact on the deployment to development mapping is expected to " + f"be minimal: you may observe a small number of unmapped files " + f"due to missing OS-specific structural patches." + ) + # Empty string forces Nix to use the container's native architecture + system_config = "" + else: + # Use crossSystem for compatible cross-architectures + system_config = ( + f'localSystem = builtins.currentSystem; crossSystem = "{system}";' + ) + + config_str = ( + "config = { " + "allowBroken = true; " + "allowUnfree = true; " + "allowUnsupportedSystem = true; " + "};" + ) + + nixpkgs_import = ( + f'import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/' + f'{commit_hash}.tar.gz") {{ {system_config} {config_str} }}' + ) + + nix_expression = ( + f"let " + f" pkgs = {nixpkgs_import}; " + f" pkg = pkgs.{name}; " + f"in " + f"pkg.overrideAttrs (old: {{ " + f' name = (old.name or "{name}") + "-patched-src"; ' + f' phases = [ "unpackPhase" "patchPhase" "installPhase" ]; ' + f' installPhase = "mkdir -p $out && cp -a . $out/"; ' + f' outputs = [ "out" ]; ' + f" separateDebugInfo = false; " + f" doCheck = false; " + f" doInstallCheck = false; " + f"}})" + ) + + container_script = f""" + OUT_PATH=$(nix-build --no-out-link -E '{nix_expression}') + if [ -z "$OUT_PATH" ] || [ ! -d "$OUT_PATH" ]; then + echo "Error: nix-build failed to return a valid store path." >&2 + exit 1 + fi + cp -a "$OUT_PATH/." /build_output/ + """ + + cmd = [ + "docker", + "run", + "--rm", + "-v", + "nix-eval-cache:/nix", + "-v", + f"{absolute_out_dir}:/build_output", + "nixos/nix", + "/bin/sh", + "-c", + container_script, + ] + + task_description = f"Nix Build & Patch ({name} for {system})" + + try: + subprocess.run(cmd, capture_output=True, text=True, check=True, timeout=600) # noqa: S603 + return str(extracted_path) + except subprocess.CalledProcessError as e: + logger.error(f"Failed: {task_description} with error: {e.stderr.strip()}") + return "" + except subprocess.TimeoutExpired: + logger.error(f"==> Failed: {task_description} with error: Process timed out") + return "" + + +def ensure_multiarch_emulation(): + """ + Configure Docker host with binfmt emulators to support + multi-architecture execution and builds. + """ + cmd = [ + "docker", + "run", + "--privileged", + "--rm", + "tonistiigi/binfmt", + "--install", + "all", + ] + try: + subprocess.run(cmd, capture_output=True, text=True, check=True, timeout=60) # noqa: S603 + return True + except subprocess.CalledProcessError as e: + logger.warning(f"Could not install binfmt multi-arch emulators: {e.stderr}") + return False + except subprocess.TimeoutExpired: + logger.warning("Timeout trying to setup binfmt emulators. Skipping.") + return False diff --git a/scanpipe/pipes/utils.py b/scanpipe/pipes/utils.py new file mode 100644 index 0000000000..b2c00b51d0 --- /dev/null +++ b/scanpipe/pipes/utils.py @@ -0,0 +1,217 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import logging +import shutil +import subprocess +from fnmatch import fnmatch + +import requests +from license_expression import Licensing + +from scanpipe.pipes import fetch +from scanpipe.pipes import flag + +logger = logging.getLogger(__name__) + + +def validate_package_license_integrity(project): + """Validate the correctness of the package license.""" + # Patterns to ignore certain resources during license validation + ignore_patterns = [ + "*test*", + "*.sh", + ] + + for package in project.discoveredpackages.all(): + package_lic = package.get_declared_license_expression() + if package_lic: + if package.type == "cargo": + # A single cargo package only has one Cargo.toml file + # meaning only one package is defined. Therefore, we don't + # need to check for the package_uid + # In addition, the package_uid is not populated to source files: + # https://github.com/aboutcode-org/scancode.io/issues/2169 + # so we set package_uid to None to consider all resources + # in the codebase for license validation. + package_uid = None + else: + package_uid = package.package_uid + resources = project.codebaseresources.has_license_expression() + detected_lic_list = collect_detected_licenses( + resources, ignore_patterns, package_uid + ) + + if detected_lic_list: + lic_exp = " AND ".join(detected_lic_list) + detected_lic_exp = str(Licensing().dedup(lic_exp)) + + if detected_lic_exp != package_lic: + package_issues = package.extra_data.get("issues", []) + + package_issues.append( + { + "issue_type": "License Mismatch", + "declared_license": package_lic, + "detected_codebase_license": detected_lic_exp, + } + ) + + package.update_extra_data({"issues": package_issues}) + + for datafile_path in package.datafile_paths: + if not datafile_path.startswith("https://"): + data_path = project.codebaseresources.get( + path=datafile_path + ) + data_path.update(status=flag.LICENSE_ISSUE) + + resource_issues = data_path.extra_data.get("issues", []) + resource_issues.append( + { + "issue_type": "License Mismatch", + "declared_license": package_lic, + "detected_codebase_license": detected_lic_exp, + } + ) + + data_path.update_extra_data({"issues": resource_issues}) + + +def contains_ignore_pattern(resource_path, ignore_patterns): + """Check if the resource path matches any of the ignore patterns.""" + for pattern in ignore_patterns: + if fnmatch(resource_path, pattern): + return True + return False + + +def filter_ignored_licenses(license_expression, licensing): + """Filter out ignored licenses from a license expression.""" + # Some licenses are not useful for validating package license + # integrity, so we ignore them. + ignored_licenses = [ + "free-unknown", + "unknown", + "unknown-license-reference", + "unknown-spdx", + ] + + if license_expression is None: + return None + + if isinstance(license_expression, licensing.Symbol): + if ( + hasattr(license_expression, "key") + and license_expression.key in ignored_licenses + ): + return None + return license_expression + + # Handle AND operations + if isinstance(license_expression, licensing.AND): + return handle_operator_expression(license_expression, licensing, licensing.AND) + + # Handle OR operations + if isinstance(license_expression, licensing.OR): + return handle_operator_expression(license_expression, licensing, licensing.OR) + + return license_expression + + +def handle_operator_expression(expression, licensing, operator): + """ + Process AND/OR operations in a license expression, filtering out + ignored licenses. + """ + args = [] + for arg in expression.args: + filtered_arg = filter_ignored_licenses(arg, licensing) + if filtered_arg is not None: + args.append(filtered_arg) + if not args: + return None + if len(args) == 1: + return args[0] + + return operator(*args) + + +def collect_detected_licenses(resources, ignore_patterns, package_uid=None): + """Collect detected licenses from resources, ignoring defined patterns.""" + licensing = Licensing() + detected_lic_list = [] + + for resource in resources: + if contains_ignore_pattern(resource.path, ignore_patterns): + continue + + # If a package_uid is provided, only consider resources linked to it + if package_uid and package_uid not in resource.for_packages: + continue + + license_str = resource.detected_license_expression + if not license_str: + continue + try: + parsed_lic = licensing.parse(license_str) + + # Filter out the ignored keys + filtered_license = filter_ignored_licenses(parsed_lic, licensing) + + if filtered_license is not None: + final_lic = str(filtered_license) + + if final_lic not in detected_lic_list: + # Apply parentheses so that the 'OR' expression will + # not be filtered out when doing deduplication later. + detected_lic_list.append(f"({final_lic})") + + except Exception: + logger.warning( + "Failed to parse the license expression: %s at %s", + license_str, + resource.path, + ) + return detected_lic_list + + +def fetch_path(purl): + """Fetch the purl and return the location of the fetched tarball""" + try: + return fetch.fetch_url(url=purl).path + except (ValueError, requests.RequestException) as e: + logger.warning("Failed to fetch package: %s - %s", purl, e) + return None + + +def check_docker_command(): + """Check if the Docker command is available and the daemon is running.""" + docker_path = shutil.which("docker") + if not docker_path: + return False + + try: + subprocess.run([docker_path, "info"], capture_output=True, check=True) # noqa: S603 + return True + except (subprocess.SubprocessError, FileNotFoundError): + return False diff --git a/scanpipe/templates/scanpipe/package_list.html b/scanpipe/templates/scanpipe/package_list.html index 90f917c245..206a66526e 100644 --- a/scanpipe/templates/scanpipe/package_list.html +++ b/scanpipe/templates/scanpipe/package_list.html @@ -34,6 +34,11 @@ {% endif %} + {% if package.extra_data.issues %} + + + + {% endif %} @@ -75,4 +80,4 @@ {% include 'scanpipe/includes/pagination.html' with page_obj=page_obj %} {% endif %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/scanpipe/tests/pipes/test_nix.py b/scanpipe/tests/pipes/test_nix.py new file mode 100644 index 0000000000..bafec51bef --- /dev/null +++ b/scanpipe/tests/pipes/test_nix.py @@ -0,0 +1,355 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +import tempfile +from pathlib import Path +from unittest import mock + +from django.test import TestCase + +from packageurl import PackageURL + +from scanpipe.pipes import nix + + +class ScanPipeNixPipesTest(TestCase): + data = Path(__file__).parent.parent / "data" + + def test_scanpipe_nix_check_input_and_return_purl(self): + project = mock.Mock() + project.inputsources.all.return_value = [ + "pkg:nix/nixpkgs/hello@2.12.1?system=x86_64-linux&commit=1234abcd" + ] + + expected = PackageURL.from_string( + "pkg:nix/nixpkgs/hello@2.12.1?system=x86_64-linux&commit=1234abcd" + ) + result = nix.check_input_and_return_purl(project) + self.assertEqual(result, expected) + + def test_scanpipe_nix_check_input_and_return_purl_no_input(self): + project = mock.Mock() + project.inputsources.all.return_value = [] + with self.assertRaisesMessage(ValueError, "Only 1 nix purl is accepted."): + nix.check_input_and_return_purl(project) + + def test_scanpipe_nix_check_input_and_return_purl_multi_input(self): + project = mock.Mock() + project.inputsources.all.return_value = [ + "pkg:nix/nixpkgs/hello@2.12.1?system=x86_64-linux", + "pkg:nix/nixpkgs/world@2.40.0?system=x86_64-linux", + ] + with self.assertRaisesMessage(ValueError, "Only 1 nix purl is accepted."): + nix.check_input_and_return_purl(project) + + def test_scanpipe_nix_check_input_and_return_purl_non_supported_type(self): + project = mock.Mock() + project.inputsources.all.return_value = ["pkg:npm/test@1.0"] + with self.assertRaisesMessage(ValueError, "Only nix purl is supported."): + nix.check_input_and_return_purl(project) + + def test_scanpipe_nix_check_input_and_return_purl_invalid_namespace(self): + project = mock.Mock() + project.inputsources.all.return_value = [ + "pkg:nix/namespace/hello@2.12.1?system=x86_64-linux" + ] + with self.assertRaisesMessage( + Exception, "Only official nixpkgs repository is supported" + ): + nix.check_input_and_return_purl(project) + + def test_scanpipe_nix_check_input_and_return_purl_missing_version_and_commit(self): + project = mock.Mock() + project.inputsources.all.return_value = [ + "pkg:nix/nixpkgs/hello?system=x86_64-linux" + ] + with self.assertRaisesMessage( + Exception, "Version or a 'commit' qualifier is required." + ): + nix.check_input_and_return_purl(project) + + def test_scanpipe_nix_check_input_and_return_purl_missing_system(self): + project = mock.Mock() + project.inputsources.all.return_value = ["pkg:nix/nixpkgs/hello@2.12.1"] + with self.assertRaisesMessage( + Exception, + "The 'system' qualifier is required to resolve system-specific binaries.", + ): + nix.check_input_and_return_purl(project) + + @mock.patch("scanpipe.pipes.nix.fetch_json_response") + def test_scanpipe_nix_get_package_data(self, mock_fetch_json): + mock_fetch_json.return_value = { + "releases": [ + { + "version": "2.12.1", + "platforms": [ + { + "arch": "x86-64", + "os": "Linux", + "system": "x86_64-linux", + "commit_hash": "1234abcd", + "outputs": [ + { + "name": "out", + "path": "/nix/store/aaaaaaa-hello-2.12.1", + } + ], + } + ], + "platforms_summary": "Linux only", + "outputs_summary": "out", + } + ] + } + purl = PackageURL.from_string( + "pkg:nix/nixpkgs/hello@2.12.1?system=x86_64-linux" + ) + + result = nix.get_package_data(purl) + self.assertEqual( + result, + { + "releases": [ + { + "version": "2.12.1", + "platforms": [ + { + "arch": "x86-64", + "os": "Linux", + "system": "x86_64-linux", + "commit_hash": "1234abcd", + "outputs": [ + { + "name": "out", + "path": "/nix/store/aaaaaaa-hello-2.12.1", + } + ], + } + ], + "platforms_summary": "Linux only", + "outputs_summary": "out", + } + ] + }, + ) + mock_fetch_json.assert_called_once_with( + "https://search.devbox.sh/v2/pkg?name=hello" + ) + + def test_scanpipe_nix_get_commit_hash_nix_store_path(self): + data = { + "releases": [ + { + "version": "2.12.1", + "platforms": [ + { + "system": "x86_64-linux", + "commit_hash": "1234abcd", + "outputs": [ + { + "name": "out", + "path": "/nix/store/aaaaaaa-hello-2.12.1", + }, + { + "name": "debug", + "path": "/nix/store/aaaaaaa-hello-2.12.1-debug", + }, + ], + } + ], + } + ] + } + + commit, store_path = nix.get_commit_hash_nix_store_path( + data, "x86_64-linux", "out", "2.12.1", "1234abcd" + ) + self.assertEqual(commit, "1234abcd") + self.assertEqual(store_path, "/nix/store/aaaaaaa-hello-2.12.1") + + @mock.patch("scanpipe.pipes.nix.subprocess.run") + def test_scanpipe_nix_get_nix_store_path_with_nix(self, mock_subprocess_run): + mock_result = mock.Mock() + mock_result.stdout = "/nix/store/evaluated-path-out" + mock_subprocess_run.return_value = mock_result + + path = nix.get_nix_store_path_with_nix( + "hello", "x86_64-linux", "out", "1234abcd" + ) + self.assertEqual(path, "/nix/store/evaluated-path-out") + + @mock.patch("scanpipe.pipes.nix.get_narinfo_url") + def test_scanpipe_nix_get_nix_download_url(self, mock_get_narinfo): + mock_get_narinfo.return_value = "nar/abc.nar.xz" + store_path = "/nix/store/aaaaaaaaaaaaa-hello-2.12.1" + + url = nix.get_nix_download_url(store_path) + self.assertEqual(url, "https://cache.nixos.org/nar/abc.nar.xz") + + @mock.patch("scanpipe.pipes.nix.requests.get") + def test_scanpipe_nix_get_narinfo_url(self, mock_requests_get): + mock_response = mock.Mock() + mock_response.text = "StorePath: /nix/store/xyz\nURL: nar/123.nar.xz" + mock_requests_get.return_value = mock_response + + url_path = nix.get_narinfo_url("https://cache.nixos.org/aaaaaaaaaaa.narinfo") + self.assertEqual(url_path, "nar/123.nar.xz") + + @mock.patch("scanpipe.pipes.nix.get_package_data") + @mock.patch("scanpipe.pipes.nix.get_nix_store_path_with_nix") + @mock.patch("scanpipe.pipes.nix.get_nix_download_url") + @mock.patch("scanpipe.pipes.nix.get_patched_source_with_docker") + @mock.patch("scanpipe.pipes.utils.fetch_path") + def test_scanpipe_nix_fetch_inputs( + self, + mock_fetch_path, + mock_get_patched_source, + mock_get_download_url, + mock_get_store_path_with_nix, + mock_get_package_data, + ): + mock_get_package_data.return_value = None + mock_get_store_path_with_nix.return_value = "/nix/store/aaaaaaaaaa" + + mock_get_download_url.return_value = "https://cache.nixos.org/nar/hello.nar.xz" + mock_get_patched_source.return_value = "/path/extracted/from" + mock_fetch_path.return_value = "/path/debug/to" + + purl = PackageURL.from_string( + "pkg:nix/nixpkgs/hello@2.12.1?system=x86_64-linux&commit=1234abcd" + ) + + with tempfile.TemporaryDirectory() as temp_dir: + src_path, bin_path, output_fmt, error_msg, warning_msg = nix.fetch_inputs( + purl, temp_dir + ) + + self.assertEqual(src_path, "/path/extracted/from") + self.assertEqual(bin_path, "/path/debug/to") + self.assertEqual(output_fmt, "debug") + self.assertEqual(error_msg, "") + self.assertEqual(warning_msg, "") + + mock_get_store_path_with_nix.assert_called_once() + + @mock.patch("scanpipe.pipes.nix.get_package_data") + @mock.patch("scanpipe.pipes.nix.get_nix_store_path_with_nix") + @mock.patch("scanpipe.pipes.nix.get_nix_download_url") + @mock.patch("scanpipe.pipes.nix.get_patched_source_with_docker") + @mock.patch("scanpipe.pipes.nix.build_binary_with_docker") + @mock.patch("scanpipe.pipes.utils.fetch_path") + def test_scanpipe_nix_fetch_inputs_fallback_build( + self, + mock_fetch_path, + mock_build_binary, + mock_get_patched_source, + mock_get_download_url, + mock_get_store_path_with_nix, + mock_get_package_data, + ): + """Test that fetch_inputs falls back to local build if download fails.""" + mock_get_package_data.return_value = None + mock_get_store_path_with_nix.return_value = "/nix/store/aaaaaaaaaa" + + # Simulate a missing/failed cache download + mock_get_download_url.return_value = "" + mock_fetch_path.return_value = "" + + # Simulate a successful local build and source extraction + mock_build_binary.return_value = "/path/built/locally/to" + mock_get_patched_source.return_value = "/path/extracted/from" + + purl = PackageURL.from_string( + "pkg:nix/nixpkgs/hello@2.12.1?system=x86_64-linux&commit=1234abcd" + ) + + with tempfile.TemporaryDirectory() as temp_dir: + src_path, bin_path, output_fmt, error_msg, warning_msg = nix.fetch_inputs( + purl, temp_dir + ) + + self.assertEqual(src_path, "/path/extracted/from") + self.assertEqual(bin_path, "/path/built/locally/to") + self.assertEqual(output_fmt, "debug") + self.assertEqual(error_msg, "") + self.assertTrue("Built locally using commit" in warning_msg) + + mock_build_binary.assert_called_once() + mock_get_store_path_with_nix.assert_called_once() + + @mock.patch("scanpipe.pipes.nix.get_commit_hash_nix_store_path") + def test_scanpipe_nix_get_nix_store_path_success( + self, mock_get_commit_hash_nix_store_path + ): + mock_get_commit_hash_nix_store_path.return_value = ( + "1234abcd", + "/nix/store/hello-path", + ) + + output_fmt, path, commit = nix.get_nix_store_path( + data={"releases": []}, + name="hello", + version="2.12.1", + system="x86_64-linux", + commit_hash="1234abcd", + user_output="", + ) + + self.assertEqual(output_fmt, "debug") + self.assertEqual(path, "/nix/store/hello-path") + self.assertEqual(commit, "1234abcd") + + @mock.patch("scanpipe.pipes.nix.subprocess.run") + def test_scanpipe_nix_get_patched_source_with_docker_success( + self, mock_subprocess_run + ): + """Test successful fetching and patching of source using Docker.""" + mock_subprocess_run.return_value = mock.Mock(returncode=0) + + with tempfile.TemporaryDirectory() as temp_dir: + result = nix.get_patched_source_with_docker( + name="hello", + output_dir=temp_dir, + system="x86_64-linux", + commit_hash="1234abcd", + ) + + expected_path = str(Path(temp_dir) / "from") + self.assertEqual(result, expected_path) + mock_subprocess_run.assert_called_once() + + @mock.patch("scanpipe.pipes.nix.subprocess.run") + def test_scanpipe_nix_extract_nar_archive_success(self, mock_subprocess_run): + """Test extracting a .nar archive via Docker.""" + mock_subprocess_run.return_value = mock.Mock(returncode=0) + + with tempfile.TemporaryDirectory() as temp_dir: + # We don't actually need the file to exist for the mocked test + archive_path = Path(temp_dir) / "hello-bin.nar.xz" + + result = nix.extract_nar_archive( + archive_path=str(archive_path), output_dir=temp_dir, output="debug" + ) + + expected_extracted_path = str(Path(temp_dir).resolve() / "to" / "debug") + self.assertEqual(result, expected_extracted_path) diff --git a/scanpipe/tests/pipes/test_utils.py b/scanpipe/tests/pipes/test_utils.py new file mode 100644 index 0000000000..490f7a0997 --- /dev/null +++ b/scanpipe/tests/pipes/test_utils.py @@ -0,0 +1,174 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + + +from unittest import mock + +from django.test import TestCase + +from license_expression import Licensing + +from scanpipe.pipes import flag +from scanpipe.pipes import utils + + +class ScanPipeUtilsTest(TestCase): + def setUp(self): + self.licensing = Licensing() + + @mock.patch("scanpipe.models.CodebaseResource") + @mock.patch("scanpipe.models.DiscoveredPackage") + @mock.patch("scanpipe.models.Project") + def test_validate_package_license_integrity_mismatch( + self, mock_project_class, mock_package_class, mock_resource_class + ): + mock_project = mock_project_class() + mock_package = mock_package_class() + + mock_package.type = "pypi" + mock_package.package_uid = "pkg:pypi/test@1.0" + mock_package.get_declared_license_expression.return_value = "mit" + mock_package.datafile_paths = ["src/main.py"] + mock_package.extra_data = {} + + mock_project.discoveredpackages.all.return_value = [mock_package] + + mock_resource = mock_resource_class() + mock_resource.path = "src/main.py" + mock_resource.for_packages = ["pkg:pypi/test@1.0"] + mock_resource.detected_license_expression = "gpl-3.0" + + mock_project.codebaseresources.has_license_expression.return_value = [ + mock_resource + ] + + mock_data_path = mock_resource_class() + mock_data_path.extra_data = {} + mock_project.codebaseresources.get.return_value = mock_data_path + + utils.validate_package_license_integrity(mock_project) + + package_update_args = mock_package.update_extra_data.call_args.args[0] + self.assertEqual( + package_update_args["issues"][0]["issue_type"], "License Mismatch" + ) + self.assertEqual( + package_update_args["issues"][0]["detected_codebase_license"], "gpl-3.0" + ) + + mock_data_path.update.assert_called_once_with(status=flag.LICENSE_ISSUE) + + def test_contains_ignore_pattern(self): + ignore_patterns = ["*test*", "*.sh"] + self.assertTrue( + utils.contains_ignore_pattern("src/test_main.py", ignore_patterns) + ) + self.assertTrue( + utils.contains_ignore_pattern("scripts/build.sh", ignore_patterns) + ) + self.assertFalse(utils.contains_ignore_pattern("src/main.py", ignore_patterns)) + + def test_filter_ignored_licenses(self): + exp1 = self.licensing.parse("mit") + self.assertEqual( + str(utils.filter_ignored_licenses(exp1, self.licensing)), "mit" + ) + + exp2 = self.licensing.parse("unknown") + self.assertIsNone(utils.filter_ignored_licenses(exp2, self.licensing)) + + exp3 = self.licensing.parse("mit AND unknown") + self.assertEqual( + str(utils.filter_ignored_licenses(exp3, self.licensing)), "mit" + ) + + exp4 = self.licensing.parse("unknown-spdx OR free-unknown") + self.assertIsNone(utils.filter_ignored_licenses(exp4, self.licensing)) + + def test_collect_detected_licenses(self): + mock_resource1 = mock.Mock() + mock_resource1.path = "src/main.py" + mock_resource1.for_packages = ["pkg:pypi/test@1.0"] + mock_resource1.detected_license_expression = "mit AND unknown" + + mock_resource2 = mock.Mock() + mock_resource2.path = "test/test_main.py" + mock_resource2.for_packages = ["pkg:pypi/test@1.0"] + mock_resource2.detected_license_expression = "gpl-3.0" + + mock_resource3 = mock.Mock() + mock_resource3.path = "src/other.py" + mock_resource3.for_packages = ["pkg:pypi/test@2.0"] + mock_resource3.detected_license_expression = "apache-2.0" + + resources = [mock_resource1, mock_resource2, mock_resource3] + ignore_patterns = ["*test*"] + + result = utils.collect_detected_licenses( + resources, ignore_patterns, package_uid="pkg:pypi/test@1.0" + ) + + self.assertEqual(result, ["(mit)"]) + + def test_handle_operator_expression_and(self): + expr = self.licensing.parse("mit AND apache-2.0") + result = utils.handle_operator_expression( + expr, self.licensing, self.licensing.AND + ) + self.assertEqual(str(result), "mit AND apache-2.0") + + def test_handle_operator_expression_or(self): + expr = self.licensing.parse("mit OR bsd-3-clause") + result = utils.handle_operator_expression( + expr, self.licensing, self.licensing.OR + ) + self.assertEqual(str(result), "mit OR bsd-3-clause") + + def test_handle_operator_expression_filters_to_single_arg(self): + # 'unknown' gets filtered out to None, leaving only 'mit' (len == 1) + expr = self.licensing.parse("mit AND unknown") + result = utils.handle_operator_expression( + expr, self.licensing, self.licensing.AND + ) + self.assertEqual(str(result), "mit") + + def test_handle_operator_expression_all_filtered_out(self): + # Both 'unknown' and 'free-unknown' get filtered out, leaving empty args + expr = self.licensing.parse("unknown AND free-unknown") + result = utils.handle_operator_expression( + expr, self.licensing, self.licensing.AND + ) + self.assertIsNone(result) + + @mock.patch("scanpipe.pipes.utils.shutil.which") + @mock.patch("scanpipe.pipes.utils.subprocess.run") + def test_check_docker_command_success(self, mock_subprocess_run, mock_shutil_which): + mock_shutil_which.return_value = "/usr/bin/docker" + mock_subprocess_run.return_value = mock.Mock(returncode=0) + + self.assertTrue(utils.check_docker_command()) + + @mock.patch("scanpipe.pipes.utils.shutil.which") + def test_check_docker_command_not_found(self, mock_shutil_which): + mock_shutil_which.return_value = None + + self.assertFalse(utils.check_docker_command()) diff --git a/scanpipe/views.py b/scanpipe/views.py index 7981f2a7d7..a22278220c 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -1774,6 +1774,7 @@ def get_queryset(self): "compliance_alert", "copyright", "affected_by_vulnerabilities", + "extra_data", ) .with_resources_count() .order_by_package_url()