[Feature] : API ENDPOINTS PR 7 : Test Artifacts and Cleanups

.github/workflows/ci-vm-scripts.yml

@@ -0,0 +1,37 @@
+name: CI-VM scripts
+
+# Static analysis for the CI-VM runner scripts (the scripts that run on the
+# throwaway test VMs). Runs only when those scripts change. shellcheck lints the
+# bash runners; PSScriptAnalyzer lints the PowerShell startup script.
+
+on:
+  pull_request:
+    paths:
+      - 'install/ci-vm/**'
+      - '.github/workflows/ci-vm-scripts.yml'
+  push:
+    paths:
+      - 'install/ci-vm/**'
+      - '.github/workflows/ci-vm-scripts.yml'
+
+jobs:
+  shellcheck:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install shellcheck
+        run: sudo apt-get update && sudo apt-get install -y shellcheck
+      - name: shellcheck (bash runner scripts)
+        run: shellcheck --severity=error install/ci-vm/ci-linux/ci/runCI install/ci-vm/ci-linux/startup-script.sh
+
+  psscriptanalyzer:
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: PSScriptAnalyzer (startup-script.ps1)
+        shell: pwsh
+        run: |
+          Set-PSRepository PSGallery -InstallationPolicy Trusted
+          Install-Module -Name PSScriptAnalyzer -Force -Scope CurrentUser
+          $issues = Invoke-ScriptAnalyzer -Path install/ci-vm/ci-windows/startup-script.ps1 -Severity Error
+          if ($issues) { $issues | Format-Table -AutoSize; exit 1 }

.pycodestylerc

@@ -1,5 +1,5 @@
 [pycodestyle]
 count = True
 max-line-length = 120
-exclude=test_diff.py,migrations,venv*,parse.py,config.py
-ignore = E701
+exclude=test_diff.py,migrations,venv*,.venv*,parse.py,config.py
+ignore = E701,W503

install/ci-vm/ci-linux/ci/runCI

@@ -103,6 +103,21 @@ function executeCommand {
         fi
 }
 
+# Send a build artifact to the server (best-effort: a failed upload must never
+# abort the run, so this always returns 0). Reuses the reportURL/curl pattern.
+function sendArtifact {
+    local artifactType="$1"
+    local filePath="$2"
+    if [ ! -f "${filePath}" ]; then
+        echo "Artifact ${artifactType}: no file at '${filePath}', skipping" >> "${logFile}"
+        return 0
+    fi
+    echo "Uploading ${artifactType} artifact (${filePath})" >> "${logFile}"
+    curl -s -A "${userAgent}" --form "type=artifactupload" --form "artifact_type=${artifactType}" \
+        --form "file=@${filePath}" "${reportURL}" >> "${logFile}" 2>&1
+    return 0
+}
+
 # Source variables
 . "$DIR/variables"
 
@@ -125,7 +140,25 @@ if [ -e "${dstDir}/ccextractor" ]; then
         echo "=== End Version Info ===" >> "${logFile}"
         postStatus "testing" "Running tests"
         executeCommand cd ${suiteDstDir}
-        executeCommand ${tester} --debug --entries "${testFile}" --executable "ccextractor" --tempfolder "${tempFolder}" --timeout 600 --reportfolder "${reportFolder}" --resultfolder "${resultFolder}" --samplefolder "${sampleFolder}" --method Server --url "${reportURL}"
+
+        # Enable core dumps and capture the test run's combined stdout/stderr so
+        # both can be uploaded as artifacts (the API serves them per run).
+        ulimit -c unlimited 2>/dev/null
+        echo "core.%p" | sudo tee /proc/sys/kernel/core_pattern >/dev/null 2>&1
+        combinedLog="${reportFolder}/combined_stdout.log"
+        ${tester} --debug --entries "${testFile}" --executable "ccextractor" --tempfolder "${tempFolder}" --timeout 600 --reportfolder "${reportFolder}" --resultfolder "${resultFolder}" --samplefolder "${sampleFolder}" --method Server --url "${reportURL}" > "${combinedLog}" 2>&1
+        testerStatus=$?
+        cat "${combinedLog}" >> "${logFile}"
+
+        # Upload artifacts before any failure-halt so crash data survives.
+        sendArtifact "binary" "${dstDir}/ccextractor"
+        sendArtifact "combined_stdout" "${combinedLog}"
+        sendArtifact "coredump" "$(ls -1t core.* 2>/dev/null | head -n1)"
+
+        if [ ${testerStatus} -ne 0 ]; then
+                haltAndCatchFire ""
+        fi
+
         sendLogFile
         postStatus "completed" "Ran all tests"
 

install/ci-vm/ci-linux/startup-script.sh

@@ -4,7 +4,7 @@ curl -L -O https://github.com/GoogleCloudPlatform/gcsfuse/releases/download/v3.2
 dpkg --install gcsfuse_3.2.0_amd64.deb
 rm gcsfuse_3.2.0_amd64.deb
 
-apt install gnupg ca-certificates
+apt install -y gnupg ca-certificates
 gpg --homedir /tmp --no-default-keyring --keyring /usr/share/keyrings/mono-official-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
 echo "deb [signed-by=/usr/share/keyrings/mono-official-archive-keyring.gpg] https://download.mono-project.com/repo/ubuntu stable-focal main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list
 sudo apt update
@@ -14,7 +14,8 @@ mkdir repository
 cd repository
 
 # Use gcsfuse and import required files
-mkdir temp TestFiles TestResults vm_data reports
+# TempFiles is used by the tester (--tempfolder) and must exist
+mkdir temp TestFiles TestResults TempFiles vm_data reports
 
 gcs_bucket=$(curl http://metadata/computeMetadata/v1/instance/attributes/bucket -H "Metadata-Flavor: Google")
 
@@ -31,6 +32,9 @@ mount vm_data
 mount TestFiles
 mount TestResults
 
+# Give gcsfuse mounts time to become ready
+sleep 10
+
 cp temp/* ./
 
 chmod +x bootstrap

install/ci-vm/ci-windows/ci/runCI.bat

@@ -15,7 +15,10 @@ for /F %%R in ('curl http://metadata/computeMetadata/v1/instance/attributes/repo
 SET userAgent="CCX/CI_BOT"
 SET logFile="%reportFolder%/log.html"
 
-call :postStatus "preparation" "Loaded variables, created log file and checking for CCExtractor build artifact" >> "%logFile%"
+rem NB: no outer ">> %logFile%" here. postStatus already appends to %logFile%
+rem internally; an outer redirect to the same file self-locks on Windows
+rem (the inner append cannot open a file the outer redirect holds open).
+call :postStatus "preparation" "Loaded variables, created log file and checking for CCExtractor build artifact"
 
 echo Checking for CCExtractor build artifact
 if EXIST "%dstDir%\ccextractorwinfull.exe" (
@@ -27,7 +30,17 @@ if EXIST "%dstDir%\ccextractorwinfull.exe" (
     echo === End Version Info === >> "%logFile%"
     call :postStatus "testing" "Running tests"
     call :executeCommand cd %suiteDstDir%
-    call :executeCommand "%tester%" --debug True --entries "%testFile%" --executable "ccextractorwinfull.exe" --tempfolder "%tempFolder%" --timeout 600 --reportfolder "%reportFolder%" --resultfolder "%resultFolder%" --samplefolder "%sampleFolder%" --method Server --url "%reportURL%"
+
+    rem Capture the test run's combined stdout/stderr for upload as an artifact.
+    "%tester%" --debug True --entries "%testFile%" --executable "ccextractorwinfull.exe" --tempfolder "%tempFolder%" --timeout 600 --reportfolder "%reportFolder%" --resultfolder "%resultFolder%" --samplefolder "%sampleFolder%" --method Server --url "%reportURL%" > "%reportFolder%/combined_stdout.log" 2>&1
+    if errorlevel 1 (set "testerFailed=1") else (set "testerFailed=")
+    type "%reportFolder%/combined_stdout.log" >> "%logFile%"
+
+    rem Upload artifacts (best-effort; never aborts the run). Windows skips coredump for v1.
+    call :sendArtifact "binary" "%dstDir%\ccextractorwinfull.exe"
+    call :sendArtifact "combined_stdout" "%reportFolder%/combined_stdout.log"
+
+    if defined testerFailed call :haltAndCatchFire ""
 
     call :sendLogFile
 
@@ -144,3 +157,18 @@ if !sl_attempt! LEQ %sl_max_retries% (
 echo ERROR: Failed to upload log after %sl_max_retries% attempts >> "%logFile%"
 endlocal
 EXIT /B 1
+
+rem Send a build artifact to the server (best-effort; never aborts the run)
+:sendArtifact
+setlocal
+set "sa_type=%~1"
+set "sa_file=%~2"
+if NOT EXIST "%sa_file%" (
+    echo Artifact %sa_type%: no file at "%sa_file%", skipping >> "%logFile%"
+    endlocal
+    EXIT /B 0
+)
+echo Uploading %sa_type% artifact (%sa_file%) >> "%logFile%"
+curl -s -A "%userAgent%" --form "type=artifactupload" --form "artifact_type=%sa_type%" --form "file=@%sa_file%" "%reportURL%" >> "%logFile%" 2>&1
+endlocal
+EXIT /B 0

migrations/versions/d4f8e2a1b3c7_.py

@@ -0,0 +1,44 @@
+"""Add api_token table for scoped API token auth.
+
+Revision ID: d4f8e2a1b3c7
+Revises: c8f3a2b1d4e5
+Create Date: 2026-06-11 03:00:00.000000
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = 'd4f8e2a1b3c7'
+down_revision = 'c8f3a2b1d4e5'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    """Apply the migration."""
+    op.add_column('user', sa.Column('github_login', sa.String(length=255), nullable=True))
+    op.create_table(
+        'api_token',
+        sa.Column('id', sa.Integer(), nullable=False, autoincrement=True),
+        sa.Column('user_id', sa.Integer(), nullable=False),
+        sa.Column('token_name', sa.String(length=50), nullable=False),
+        sa.Column('token_hash', sa.String(length=255), nullable=False),
+        sa.Column('token_prefix', sa.String(length=16), nullable=False),
+        sa.Column('scopes_json', sa.Text(), nullable=False),
+        sa.Column('created_at', sa.DateTime(timezone=True), nullable=False),
+        sa.Column('expires_at', sa.DateTime(timezone=True), nullable=False),
+        sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True),
+        sa.PrimaryKeyConstraint('id'),
+        sa.ForeignKeyConstraint(['user_id'], ['user.id'], onupdate='CASCADE', ondelete='CASCADE'),
+        sa.UniqueConstraint('user_id', 'token_name', name='uq_user_token_name'),
+        mysql_engine='InnoDB'
+    )
+    op.create_index('ix_api_token_token_prefix', 'api_token', ['token_prefix'])
+
+
+def downgrade():
+    """Revert the migration."""
+    op.drop_index('ix_api_token_token_prefix', table_name='api_token')
+    op.drop_table('api_token')
+    op.drop_column('user', 'github_login')

mod_api/__init__.py

@@ -0,0 +1,43 @@
+"""
+mod_api: JSON REST API blueprint for the CCExtractor CI platform.
+
+Registered at /api/v1. All endpoints return structured JSON, use scoped
+Bearer token auth, and enforce per-client rate limiting.
+"""
+
+from flask import Blueprint
+
+mod_api = Blueprint('api', __name__)
+
+# Middleware imports
+from mod_api.middleware import auth  # noqa: E402
+from mod_api.middleware import error_handler  # noqa: E402
+from mod_api.middleware import rate_limit  # noqa: E402
+from mod_api.middleware import security  # noqa: E402
+
+# Explicitly register before_request hooks in the exact order they should run
+mod_api.before_request(auth.authenticate_request)
+mod_api.before_request(rate_limit.check_rate_limit)
+mod_api.before_request(auth.enforce_auth_error)
+
+# Explicitly register after_request hooks.
+# NOTE: Flask executes after_request hooks in REVERSE registration order.
+# Registration:  security → rate_limit → (convert is app-level, see below)
+# Execution:     rate_limit → security
+# This means rate-limit headers are added first, then security headers layer
+# on top — both on the same response object.
+mod_api.after_request(security.add_security_headers)
+mod_api.after_request(rate_limit.add_rate_limit_headers)
+
+# Registered as after_app_request so it fires for ALL requests (including
+# routing-level 404s/405s that never enter the blueprint).
+mod_api.after_app_request(error_handler.convert_api_errors_to_json)
+
+# Route modules
+from mod_api.routes import auth as auth_routes  # noqa: E402, F401
+from mod_api.routes import \
+    errors_logs as errors_logs_routes  # noqa: E402, F401
+from mod_api.routes import results as results_routes  # noqa: E402, F401
+from mod_api.routes import runs as runs_routes  # noqa: E402, F401
+from mod_api.routes import samples as samples_routes  # noqa: E402, F401
+from mod_api.routes import system as system_routes  # noqa: E402, F401

mod_api/middleware/__init__.py

@@ -0,0 +1 @@
+"""mod_api.middleware: auth, rate limiting, validation, and error handling."""