Semgrep - Improve pre scan checks and ease of extending rules

scanners/boostsecurityio/semgrep/module.yaml

@@ -12,29 +12,22 @@ config:
   - .semgrep/*
 
 setup:
-  - name: Validate rules
-    environment:
-      SEMGREP_RULES: ${SEMGREP_RULES:-https://assets.build.boostsecurity.io/semgrep-rules/stable/all-sast-rules.yml}
+  - name: Utility scripts
     run: |
-      echo "SEMGREP_RULES set to: '$SEMGREP_RULES'"
-      for rule in $SEMGREP_RULES; do
-        case "$rule" in
-          .semgrep/*|http://*|https://*)
-            # valid rule token; do nothing
-            ;;
-          *)
-            echo "Semgrep Community Rules cannot be used. Provide a URL or relative path to rules file or leave blank for Boost curated rules."
-            exit 1
-            ;;
-        esac
-      done
+      mkdir -p $SETUP_PATH/pre-scan-checks/
+      cp $SETUP_PATH/../../registry/scanners/boostsecurityio/semgrep/prescan_checks.sh $SETUP_PATH/pre-scan-checks/semgrep
 
 steps:
+  - run: |
+      $SETUP_PATH/pre-scan-checks/semgrep
+    environment:
+      SEMGREP_RULES: ${SEMGREP_RULES:-boost/sast/rules/semgrep@stable}
   - scan:
       command:
         docker:
           image: returntocorp/semgrep:1.114.0@sha256:0cd75960cfec2215ff734a4f6379bbbb6edb82de0c24593dd0a70ec65e9860a9
-          command: semgrep scan --oss-only --sarif --quiet --disable-version-check --metrics=off .
+          command: |
+            semgrep scan --config ./.semgrep --oss-only --sarif --quiet --disable-version-check --metrics=off .
           workdir: /src
           environment:
             XDG_CONFIG_HOME: /tmp

scanners/boostsecurityio/semgrep/prescan_checks.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+local_rules_dst=$(mktemp -d)
+if [ -d ".semgrep" ]
+then
+    mv .semgrep/ "$local_rules_dst"
+fi
+mkdir -p .semgrep
+
+fetch_remote() {
+    file_name="${1##*/}"
+    file_name=$(echo $file_name | cut -d '#' -f 1 | cut -d '?' -f 1)
+    file_extension="${file_name##*.}"
+    if [ "$file_extension" != "yaml" ] && [ "$file_extension" != "yml" ]
+    then
+        >&2 echo "Semgrep custom rules validation failed."
+        >&2 echo " The provided URL does not point to a YAML file: $1."
+        rm -rf $local_rules_dst
+        exit 1
+    fi
+
+    dst_file=$(mktemp --suffix=.yml)
+    http_code=$(curl -s -L --fail -w '%{http_code}\n' -o $dst_file $1)
+    if [ "${http_code}" == "200" ]
+    then
+        cp $dst_file .semgrep/
+        rm -f $dst_file
+        return 0
+    fi
+
+    >&2 echo "Semgrep custom rules - Cannot fetch $1."
+    rm -f $dst_file
+    rm -rf $local_rules_dst
+    exit 1
+}
+
+SEMGREP_RULES=${SEMGREP_RULES:-boost/sast/rules/semgrep@stable}
+for rule in $SEMGREP_RULES; do
+    case "$rule" in
+      .semgrep/*)
+        # Local rules are allowed
+        if [ "$rule" == ".semgrep/*" ]
+        then
+          # 
+          cp -R -f $local_rules_dst/.semgrep/* .semgrep || true
+        else
+          if [ ! -f "$local_rules_dst/$rule" ]  && [ ! -d  "$local_rules_dst/$rule" ]
+          then
+            >&2 echo "Semgrep custom rules validation failed."
+            >&2 echo " The specific file or directory does not exist in the code repository: $rule."
+            exit 1
+          fi
+          cp -R -f "$local_rules_dst/$rule" .semgrep || true
+        fi
+        ;;
+      http://*|https://*)
+        fetch_remote $rule
+        ;;
+      boost/sast/rules/semgrep@*)
+        # Boost
+        version=$(echo "$rule" | cut -d '@' -f 2)
+        # $version is not sanitized since one can provide any URL in the boost config
+        fetch_remote "https://assets.build.boostsecurity.io/semgrep-rules/$version/all-sast-rules.yml"
+        ;;
+      *)
+        >&2 echo "Semgrep custom rules validation failed on $rule."
+        >&2 echo "  Community rules cannot be used."
+        >&2 echo "  Provide a URL or relative path to rules file or leave blank for Boost curated rules."
+        rm -rf $local_rules_dst
+        exit 1
+        ;;
+    esac
+done
+
+rm -rf $local_rules_dst
+
+if [ "$(find .semgrep -regex '.*\.ya?ml' | wc -l)" == "0" ]
+then
+  >&2 echo "Semgrep custom rules validation failed for $SEMGREP_RULES."
+  >&2 echo "  Missing yaml configuration files"
+  exit 1
+fi