Merge pull request #40 from trailofbits/better-ci

Improve CI jobs

.git-blame-ignore-revs

@@ -0,0 +1,2 @@
+# Formatted Semgrep YAML files using prettier
+c12bb5b7bf893466c24bdad5b234dcd3ad3b42df

.github/workflows/semgrep-rules-format.yml

@@ -0,0 +1,21 @@
+name: semgrep-rules-format
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+jobs:
+  semgrep-rules-format:
+    name: run semgrep rules format
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: "npm"
+          # Use this file as the cache-dependency-path because the dependencies
+          # are manually specified below, and not in a package-lock.json
+          cache-dependency-path: .github/workflows/semgrep-rules-format.yml
+      - run: npm install --global prettier
+      - run: prettier --check './**/*.{yaml,yml}'

.github/workflows/semgrep-rules-test.yml

@@ -5,7 +5,7 @@ on:
     branches:
       - main
 jobs:
-  build:
+  semgrep-rules-test:
     name: run semgrep rules tests
     runs-on: ubuntu-latest
     steps:
@@ -18,7 +18,15 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           python3 -m pip install semgrep
+          python3 -m pip install jsonschema pyyaml
       - name: validations
         run: semgrep --validate --config .
       - name: tests
         run: semgrep --test --test-ignore-todo
+      - name: metadata-tests
+        run: |
+          wget https://raw.githubusercontent.com/returntocorp/semgrep-rules/c5ad4bb0f4c7ee5b8cd47276b582e8bb57bd0a4d/.github/scripts/validate-metadata.py
+          wget https://raw.githubusercontent.com/returntocorp/semgrep-rules/develop/metadata-schema.yaml.schm
+          python ./validate-metadata.py -s ./metadata-schema.yaml.schm -f .
+      - name: rules-tests
+        run: semgrep --config="r/yaml.semgrep" --severity ERROR .

.github/workflows/update-semgrep-registry.yml

@@ -4,7 +4,7 @@ on:
     branches:
       - main
 jobs:
-  build:
+  update-semgrep-registry:
     name: Update semgrep.dev
     runs-on: ubuntu-latest
     steps:

go/anonymous-race-condition.yaml

@@ -1,25 +1,77 @@
 rules:
-- id: anonymous-race-condition
-  message: >-
-    Possible race condition due to memory aliasing of variable `$X`
-  languages: [go]
-  severity: ERROR
-  metadata:
-    category: security
-    cwe: "CWE-362: Concurrent Execution using Shared Resource with Improper Synchronization ('Race Condition')"
-    subcategory: [vuln]
-    confidence: MEDIUM
-    likelihood: HIGH
-    impact: MEDIUM
-    technology: [--no-technology--]
-    description: "Race conditions within anonymous goroutines"
-    references:
-      - https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables
+  - id: anonymous-race-condition
+    message: >-
+      Possible race condition due to memory aliasing of variable `$X`
+    languages: [go]
+    severity: ERROR
+    metadata:
+      category: security
+      cwe: "CWE-362: Concurrent Execution using Shared Resource with Improper Synchronization ('Race Condition')"
+      subcategory: [vuln]
+      confidence: MEDIUM
+      likelihood: HIGH
+      impact: MEDIUM
+      technology: [--no-technology--]
+      description: "Race conditions within anonymous goroutines"
+      references:
+        - https://github.com/golang/go/wiki/CommonMistakes#using-goroutines-on-loop-iterator-variables
 
-  patterns:
-    - pattern-either: 
-      - pattern: |
-          for $Y, $X := range ... {
+    patterns:
+      - pattern-either:
+          - pattern: |
+              for $Y, $X := range ... {
+                ...
+                go func(...){
+                  ...
+                  $FOO(..., $X, ...)
+                  ...
+                }(...)
+                ...
+              }
+          - pattern: |
+              for $Y, $X := range ... {
+                ...
+                go func(...){
+                  ...
+                  $FOO(..., $Y, ...)
+                  ...
+                }(...)
+                ...
+              }
+          - pattern: |
+              for $Y, $X := range ... {
+                ...
+                go func(...){
+                  ...
+                  $X(...)
+                  ...
+                }(...)
+                ...
+              }
+          - pattern: |
+              for $X:=...;$Y;$Z {
+                ...
+                go func(...) {
+                  ...
+                  $FOO(..., $X,...)
+                  ...
+                }(...)
+                ...
+              }
+          - pattern: |
+              for $Y, $X := range ... {
+                ...
+                go func(...){
+                  ...
+                  $X. ... .$M(...)
+                  ...
+                }(...)
+                ...
+              }
+      - pattern-not: |
+          for ..., $X := range ... {
+            ...
+            ..., $X := ..., $X
             ...
             go func(...){
               ...
@@ -28,77 +80,25 @@ rules:
             }(...)
             ...
           }
-      - pattern: |
-          for $Y, $X := range ... {
-            ...
-            go func(...){
-              ...
-              $FOO(..., $Y, ...)
-              ...
-            }(...)
-            ...
-          }
-      - pattern: |
-          for $Y, $X := range ... {
-            ...
-            go func(...){
+      - pattern-not: |
+          for ..., $X := range ... {
               ...
-              $X(...)
-              ...
-            }(...)
-            ...
-          }
-      - pattern: |
-          for $X:=...;$Y;$Z {
-            ...
-            go func(...) {
+              $X, ... := $X, ...
               ...
-              $FOO(..., $X,...)
+              go func(...){
+                ...
+                $FOO(..., $X, ...)
+                ...
+              }(...)
               ...
-            }(...)
-            ...
           }
-      - pattern: |
+      - pattern-not: |
           for $Y, $X := range ... {
             ...
-            go func(...){
-              ...
-              $X. ... .$M(...)
-              ...
-            }(...)
-            ...
-          }
-    - pattern-not: |
-        for ..., $X := range ... {
-          ...
-          ..., $X := ..., $X
-          ...
-          go func(...){
-            ...
-            $FOO(..., $X, ...)
-            ...
-          }(...)
-          ...
-        }
-    - pattern-not: |
-        for ..., $X := range ... {
-            ...
-            $X, ... := $X, ...
+            $Y, $X := $Y, $X
             ...
             go func(...){
               ...
-              $FOO(..., $X, ...)
-              ...
             }(...)
             ...
-        }
-    - pattern-not: |
-        for $Y, $X := range ... {
-          ...
-          $Y, $X := $Y, $X
-          ...
-          go func(...){
-            ...
-          }(...)
-          ...
-        }
+          }