fix: log the curl results, pipx for semgrep (#1)

### Explanation

_No response_

### Issue

_No response_

### Attestations

- [ ] I have read and agree to the [Code of Conduct](https://docs.nhcarrigan.com/community/coc/)
- [ ] I have read and agree to the [Community Guidelines](https://docs.nhcarrigan.com/community/guide/).
- [ ] My contribution complies with the [Contributor Covenant](https://docs.nhcarrigan.com/dev/covenant/).

### Dependencies

- [ ] I have pinned the dependencies to a specific patch version.

### Style

- [ ] I have run the linter and resolved any errors.
- [ ] My pull request uses an appropriate title, matching the conventional commit standards.
- [ ] My scope of feat/fix/chore/etc. correctly matches the nature of changes in my pull request.

### Tests

- [ ] My contribution adds new code, and I have added tests to cover it.
- [ ] My contribution modifies existing code, and I have updated the tests to reflect these changes.
- [ ] All new and existing tests pass locally with my changes.
- [ ] Code coverage remains at or above the configured threshold.

### Documentation

_No response_

### Versioning

_No response_

Reviewed-on: #1
Co-authored-by: Naomi Carrigan <commits@nhcarrigan.com>
Co-committed-by: Naomi Carrigan <commits@nhcarrigan.com>

.gitea/workflows/security.yml

@@ -23,16 +23,27 @@ jobs:
       - name: Ensure DefectDojo Product Exists
         env:
           DD_URL: ${{ secrets.DD_URL }}
-          # UPDATED: Using DD_TOKEN now
           DD_TOKEN: ${{ secrets.DD_TOKEN }}
           PRODUCT_NAME: ${{ github.repository }} 
           PRODUCT_TYPE_ID: 1 
         run: |
           sudo apt-get install jq -y > /dev/null
           
-          # Check if product exists
-          RESPONSE=$(curl -s -H "Authorization: Token $DD_TOKEN" "$DD_URL/api/v2/products/?name=$PRODUCT_NAME")
-          COUNT=$(echo "$RESPONSE" | jq -r '.count')
+          echo "Checking connection to $DD_URL..."
+          
+          # Check if product exists - capture HTTP code to debug connection issues
+          RESPONSE=$(curl --write-out "%{http_code}" --silent --output /tmp/response.json \
+            -H "Authorization: Token $DD_TOKEN" \
+            "$DD_URL/api/v2/products/?name=$PRODUCT_NAME")
+          
+          # If response is not 200, print error
+          if [ "$RESPONSE" != "200" ]; then
+            echo "::error::Failed to query DefectDojo. HTTP Code: $RESPONSE"
+            cat /tmp/response.json
+            exit 1
+          fi
+
+          COUNT=$(cat /tmp/response.json | jq -r '.count')
           
           if [ "$COUNT" = "0" ]; then
             echo "Creating product '$PRODUCT_NAME'..."
@@ -57,15 +68,34 @@ jobs:
           trivy fs . --scanners vuln,misconfig --format json --output trivy-results.json --exit-code 0
 
       - name: Upload Trivy to DefectDojo
+        env:
+          DD_URL: ${{ secrets.DD_URL }}
+          DD_TOKEN: ${{ secrets.DD_TOKEN }}
         run: |
-          curl -X POST "${{ secrets.DD_URL }}/api/v2/import-scan/" \
-            -H "Authorization: Token ${{ secrets.DD_TOKEN }}" \
+          echo "Uploading Trivy results..."
+          # Generate today's date in YYYY-MM-DD format
+          TODAY=$(date +%Y-%m-%d)
+          
+          HTTP_CODE=$(curl --write-out "%{http_code}" --output response.txt --silent -X POST "$DD_URL/api/v2/import-scan/" \
+            -H "Authorization: Token $DD_TOKEN" \
             -F "active=true" \
             -F "verified=true" \
             -F "scan_type=Trivy Scan" \
             -F "engagement_name=CI/CD Pipeline" \
             -F "product_name=${{ github.repository }}" \
-            -F "file=@trivy-results.json"
+            -F "scan_date=$TODAY" \
+            -F "auto_create_context=true" \
+            -F "file=@trivy-results.json")
+          
+          if [[ "$HTTP_CODE" != "200" && "$HTTP_CODE" != "201" ]]; then
+            echo "::error::Upload Failed with HTTP $HTTP_CODE"
+            echo "--- SERVER RESPONSE ---"
+            cat response.txt
+            echo "-----------------------"
+            exit 1
+          else
+            echo "Upload Success!"
+          fi
 
       # --- 2. GITLEAKS (Secrets) ---
       - name: Install Gitleaks
@@ -78,32 +108,70 @@ jobs:
         run: gitleaks detect --source . -v --report-path gitleaks-results.json --report-format json --no-git || true
 
       - name: Upload Gitleaks to DefectDojo
+        env:
+          DD_URL: ${{ secrets.DD_URL }}
+          DD_TOKEN: ${{ secrets.DD_TOKEN }}
         run: |
-          curl -X POST "${{ secrets.DD_URL }}/api/v2/import-scan/" \
-            -H "Authorization: Token ${{ secrets.DD_TOKEN }}" \
+          echo "Uploading Gitleaks results..."  
+          TODAY=$(date +%Y-%m-%d)
+
+          HTTP_CODE=$(curl --write-out "%{http_code}" --output response.txt --silent -X POST "$DD_URL/api/v2/import-scan/" \
+            -H "Authorization: Token $DD_TOKEN" \
             -F "active=true" \
             -F "verified=true" \
             -F "scan_type=Gitleaks Scan" \
             -F "engagement_name=CI/CD Pipeline" \
             -F "product_name=${{ github.repository }}" \
-            -F "file=@gitleaks-results.json"
+            -F "scan_date=$TODAY" \
+            -F "auto_create_context=true" \
+            -F "file=@gitleaks-results.json")
+
+          if [[ "$HTTP_CODE" != "200" && "$HTTP_CODE" != "201" ]]; then
+             echo "::error::Upload Failed with HTTP $HTTP_CODE"
+             echo "--- SERVER RESPONSE ---"
+             cat response.txt
+             echo "-----------------------"
+             exit 1
+          else
+             echo "Upload Success!"
+          fi
 
       # --- 3. SEMGREP (SAST) ---
-      - name: Install Semgrep
+      - name: Install Semgrep (via pipx)
         run: |
-          sudo apt-get install python3-pip -y
-          pip install semgrep
+          sudo apt-get install pipx -y
+          pipx install semgrep
+          # Add pipx binary path to GITHUB_PATH so next steps can see 'semgrep'
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
 
       - name: Run Semgrep
         run: semgrep scan --config=p/security-audit --config=p/owasp-top-ten --json --output semgrep-results.json . || true
 
       - name: Upload Semgrep to DefectDojo
+        env:
+          DD_URL: ${{ secrets.DD_URL }}
+          DD_TOKEN: ${{ secrets.DD_TOKEN }}
         run: |
-          curl -X POST "${{ secrets.DD_URL }}/api/v2/import-scan/" \
-            -H "Authorization: Token ${{ secrets.DD_TOKEN }}" \
+          echo "Uploading Semgrep results..."
+          TODAY=$(date +%Y-%m-%d)
+          
+          HTTP_CODE=$(curl --write-out "%{http_code}" --output response.txt --silent -X POST "$DD_URL/api/v2/import-scan/" \
+            -H "Authorization: Token $DD_TOKEN" \
             -F "active=true" \
             -F "verified=true" \
             -F "scan_type=Semgrep JSON Report" \
             -F "engagement_name=CI/CD Pipeline" \
             -F "product_name=${{ github.repository }}" \
-            -F "file=@semgrep-results.json"
+            -F "scan_date=$TODAY" \
+            -F "auto_create_context=true" \
+            -F "file=@semgrep-results.json")
+
+          if [[ "$HTTP_CODE" != "200" && "$HTTP_CODE" != "201" ]]; then
+             echo "::error::Upload Failed with HTTP $HTTP_CODE"
+             echo "--- SERVER RESPONSE ---"
+             cat response.txt
+             echo "-----------------------"
+             exit 1
+          else
+             echo "Upload Success!"
+          fi