Merge pull request #847 from agorevski/algore/cicd-descript-release-matrix

perf(ci): reduce GitHub Actions costs ~60-65% across all workflows

.github/dependabot.yml

@@ -4,13 +4,13 @@ updates:
   - package-ecosystem: cargo
     directory: "/"
     schedule:
-      interval: weekly
+      interval: monthly
     target-branch: main
-    open-pull-requests-limit: 5
+    open-pull-requests-limit: 3
     labels:
       - "dependencies"
     groups:
-      rust-minor-patch:
+      rust-all:
         patterns:
           - "*"
         update-types:
@@ -20,14 +20,31 @@ updates:
   - package-ecosystem: github-actions
     directory: "/"
     schedule:
-      interval: weekly
+      interval: monthly
     target-branch: main
-    open-pull-requests-limit: 3
+    open-pull-requests-limit: 1
     labels:
       - "ci"
       - "dependencies"
     groups:
-      actions-minor-patch:
+      actions-all:
+        patterns:
+          - "*"
+        update-types:
+          - minor
+          - patch
+
+  - package-ecosystem: docker
+    directory: "/"
+    schedule:
+      interval: monthly
+    target-branch: main
+    open-pull-requests-limit: 1
+    labels:
+      - "ci"
+      - "dependencies"
+    groups:
+      docker-all:
         patterns:
           - "*"
         update-types:

.github/workflows/ci-run.yml

@@ -41,25 +41,7 @@ jobs:
               run: ./scripts/ci/detect_change_scope.sh
 
     lint:
-        name: Lint Gate (Format + Clippy)
+        name: Lint Gate (Format + Clippy + Strict Delta)
-        needs: [changes]
-        if: needs.changes.outputs.rust_changed == 'true' && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:full'))
-        runs-on: blacksmith-2vcpu-ubuntu-2404
-        timeout-minutes: 20
-        steps:
-            - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
-              with:
-                  fetch-depth: 0
-            - uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable
-              with:
-                  toolchain: 1.92.0
-                  components: rustfmt, clippy
-            - uses: useblacksmith/rust-cache@f53e7f127245d2a269b3d90879ccf259876842d5 # v3
-            - name: Run rust quality gate
-              run: ./scripts/ci/rust_quality_gate.sh
-
-    lint-strict-delta:
-        name: Lint Gate (Strict Delta)
         needs: [changes]
         if: needs.changes.outputs.rust_changed == 'true' && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:full'))
         runs-on: blacksmith-2vcpu-ubuntu-2404
@@ -71,8 +53,10 @@ jobs:
             - uses: dtolnay/rust-toolchain@631a55b12751854ce901bb631d5902ceb48146f7 # stable
               with:
                   toolchain: 1.92.0
-                  components: clippy
+                  components: rustfmt, clippy
             - uses: useblacksmith/rust-cache@f53e7f127245d2a269b3d90879ccf259876842d5 # v3
+            - name: Run rust quality gate
+              run: ./scripts/ci/rust_quality_gate.sh
             - name: Run strict lint delta gate
               env:
                   BASE_SHA: ${{ needs.changes.outputs.base_sha }}
@@ -80,8 +64,8 @@ jobs:
 
     test:
         name: Test
-        needs: [changes, lint, lint-strict-delta]
+        needs: [changes, lint]
-        if: needs.changes.outputs.rust_changed == 'true' && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:full')) && needs.lint.result == 'success' && needs.lint-strict-delta.result == 'success'
+        if: needs.changes.outputs.rust_changed == 'true' && (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'ci:full')) && needs.lint.result == 'success'
         runs-on: blacksmith-2vcpu-ubuntu-2404
         timeout-minutes: 30
         steps:
@@ -106,8 +90,8 @@ jobs:
               with:
                   toolchain: 1.92.0
             - uses: useblacksmith/rust-cache@f53e7f127245d2a269b3d90879ccf259876842d5 # v3
-            - name: Build release binary
+            - name: Build binary (smoke check)
-              run: cargo build --release --locked --verbose
+              run: cargo build --locked --verbose
 
     docs-only:
         name: Docs-Only Fast Path
@@ -185,7 +169,7 @@ jobs:
     lint-feedback:
         name: Lint Feedback
         if: github.event_name == 'pull_request'
-        needs: [changes, lint, lint-strict-delta, docs-quality]
+        needs: [changes, lint, docs-quality]
         runs-on: blacksmith-2vcpu-ubuntu-2404
         permissions:
             contents: read
@@ -201,7 +185,7 @@ jobs:
                   RUST_CHANGED: ${{ needs.changes.outputs.rust_changed }}
                   DOCS_CHANGED: ${{ needs.changes.outputs.docs_changed }}
                   LINT_RESULT: ${{ needs.lint.result }}
-                  LINT_DELTA_RESULT: ${{ needs.lint-strict-delta.result }}
+                  LINT_DELTA_RESULT: ${{ needs.lint.result }}
                   DOCS_RESULT: ${{ needs.docs-quality.result }}
               with:
                   script: |
@@ -231,7 +215,7 @@ jobs:
     ci-required:
         name: CI Required Gate
         if: always()
-        needs: [changes, lint, lint-strict-delta, test, build, docs-only, non-rust, docs-quality, lint-feedback, workflow-owner-approval]
+        needs: [changes, lint, test, build, docs-only, non-rust, docs-quality, lint-feedback, workflow-owner-approval]
         runs-on: blacksmith-2vcpu-ubuntu-2404
         steps:
             - name: Enforce required status
@@ -276,7 +260,7 @@ jobs:
                   fi
 
                   lint_result="${{ needs.lint.result }}"
-                  lint_strict_delta_result="${{ needs.lint-strict-delta.result }}"
+                  lint_strict_delta_result="${{ needs.lint.result }}"
                   test_result="${{ needs.test.result }}"
                   build_result="${{ needs.build.result }}"
 

.github/workflows/feature-matrix.yml

@@ -1,12 +1,6 @@
 name: Feature Matrix
 
 on:
-    push:
-        branches: [main]
-        paths:
-            - "Cargo.toml"
-            - "Cargo.lock"
-            - "src/**"
     schedule:
         - cron: "30 4 * * 1" # Weekly Monday 4:30am UTC
     workflow_dispatch:
@@ -61,6 +55,3 @@ jobs:
 
             - name: Check feature combination
               run: cargo check --locked ${{ matrix.args }}
-
-            - name: Test feature combination
-              run: cargo test --locked ${{ matrix.args }}

.github/workflows/pr-auto-response.yml

@@ -15,7 +15,7 @@ jobs:
       (github.event.action == 'opened' || github.event.action == 'reopened' || github.event.action == 'labeled' || github.event.action == 'unlabeled')) ||
       (github.event_name == 'pull_request_target' &&
       (github.event.action == 'labeled' || github.event.action == 'unlabeled'))
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: ubuntu-latest
     permissions:
       contents: read
       issues: write
@@ -34,7 +34,7 @@ jobs:
             await script({ github, context, core });
   first-interaction:
     if: github.event.action == 'opened'
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: ubuntu-latest
     permissions:
       issues: write
       pull-requests: write
@@ -65,7 +65,7 @@ jobs:
 
   labeled-routes:
     if: github.event.action == 'labeled'
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: ubuntu-latest
     permissions:
       contents: read
       issues: write

.github/workflows/pr-check-stale.yml

@@ -12,7 +12,7 @@ jobs:
         permissions:
             issues: write
             pull-requests: write
-        runs-on: blacksmith-2vcpu-ubuntu-2404
+        runs-on: ubuntu-latest
         steps:
             - name: Mark stale issues and pull requests
               uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0

.github/workflows/pr-check-status.yml

@@ -2,7 +2,7 @@ name: PR Check Status
 
 on:
   schedule:
-    - cron: "15 */12 * * *"
+    - cron: "15 8 * * *" # Once daily at 8:15am UTC
   workflow_dispatch:
 
 permissions: {}
@@ -13,13 +13,13 @@ concurrency:
 
 jobs:
   nudge-stale-prs:
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: ubuntu-latest
     permissions:
       contents: read
       pull-requests: write
       issues: write
     env:
-      STALE_HOURS: "4"
+      STALE_HOURS: "48"
     steps:
       - name: Checkout repository
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4

.github/workflows/pr-intake-checks.yml

@@ -16,7 +16,7 @@ permissions:
 jobs:
     intake:
         name: Intake Checks
-        runs-on: blacksmith-2vcpu-ubuntu-2404
+        runs-on: ubuntu-latest
         timeout-minutes: 10
         steps:
             - name: Checkout repository

.github/workflows/pr-labeler.yml

@@ -25,8 +25,7 @@ permissions:
 
 jobs:
     label:
-        runs-on: blacksmith-2vcpu-ubuntu-2404
+        runs-on: ubuntu-latest
-        timeout-minutes: 10
         steps:
             - name: Checkout repository
               uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4

.github/workflows/pub-docker-img.yml

@@ -21,13 +21,8 @@ on:
         paths:
             - "Dockerfile"
             - ".dockerignore"
-            - "Cargo.toml"
+            - "docker-compose.yml"
-            - "Cargo.lock"
             - "rust-toolchain.toml"
-            - "src/**"
-            - "crates/**"
-            - "benches/**"
-            - "firmware/**"
             - "dev/config.template.toml"
             - ".github/workflows/pub-docker-img.yml"
     workflow_dispatch:

.github/workflows/sec-audit.yml

@@ -3,8 +3,20 @@ name: Sec Audit
 on:
     push:
         branches: [main]
+        paths:
+            - "Cargo.toml"
+            - "Cargo.lock"
+            - "src/**"
+            - "crates/**"
+            - "deny.toml"
     pull_request:
         branches: [main]
+        paths:
+            - "Cargo.toml"
+            - "Cargo.lock"
+            - "src/**"
+            - "crates/**"
+            - "deny.toml"
     schedule:
         - cron: "0 6 * * 1" # Weekly on Monday 6am UTC
 

.github/workflows/sec-codeql.yml

@@ -2,7 +2,7 @@ name: Sec CodeQL
 
 on:
     schedule:
-        - cron: "0 6,18 * * *" # Twice daily at 6am and 6pm UTC
+        - cron: "0 6 * * 1" # Weekly Monday 6am UTC
     workflow_dispatch:
 
 concurrency:

.github/workflows/sync-contributors.yml

@@ -17,7 +17,7 @@ permissions:
 jobs:
   update-notice:
     name: Update NOTICE with new contributors
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
         uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4

.github/workflows/test-benchmarks.yml

@@ -1,8 +1,8 @@
 name: Test Benchmarks
 
 on:
-    push:
+    schedule:
-        branches: [main]
+        - cron: "0 3 * * 1" # Weekly Monday 3am UTC
     workflow_dispatch:
 
 concurrency:
@@ -39,7 +39,7 @@ jobs:
                   path: |
                       target/criterion/
                       benchmark_output.txt
-                  retention-days: 30
+                  retention-days: 7
 
             - name: Post benchmark summary on PR
               if: github.event_name == 'pull_request'

scripts/ci/fetch_actions_data.py

@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+"""Fetch GitHub Actions workflow runs for a given date and summarize costs.
+
+Usage:
+    python fetch_actions_data.py [OPTIONS]
+
+Options:
+    --date YYYY-MM-DD   Date to query (default: yesterday)
+    --mode brief|full   Output mode (default: full)
+                        brief: billable minutes/hours table only
+                        full:  detailed breakdown with per-run list
+    --repo OWNER/NAME   Repository (default: zeroclaw-labs/zeroclaw)
+    -h, --help          Show this help message
+"""
+
+import argparse
+import json
+import subprocess
+from datetime import datetime, timedelta, timezone
+
+
+def parse_args():
+    """Parse command-line arguments."""
+    parser = argparse.ArgumentParser(
+        description="Fetch GitHub Actions workflow runs and summarize costs.",
+    )
+    yesterday = (datetime.now(timezone.utc) - timedelta(days=1)).strftime("%Y-%m-%d")
+    parser.add_argument(
+        "--date",
+        default=yesterday,
+        help="Date to query in YYYY-MM-DD format (default: yesterday)",
+    )
+    parser.add_argument(
+        "--mode",
+        choices=["brief", "full"],
+        default="full",
+        help="Output mode: 'brief' for billable hours only, 'full' for detailed breakdown (default: full)",
+    )
+    parser.add_argument(
+        "--repo",
+        default="zeroclaw-labs/zeroclaw",
+        help="Repository in OWNER/NAME format (default: zeroclaw-labs/zeroclaw)",
+    )
+    return parser.parse_args()
+
+
+def fetch_runs(repo, date_str, page=1, per_page=100):
+    """Fetch completed workflow runs for a given date."""
+    url = (
+        f"https://api.github.com/repos/{repo}/actions/runs"
+        f"?created={date_str}&per_page={per_page}&page={page}"
+    )
+    result = subprocess.run(
+        ["curl", "-sS", "-H", "Accept: application/vnd.github+json", url],
+        capture_output=True, text=True
+    )
+    return json.loads(result.stdout)
+
+
+def fetch_jobs(repo, run_id):
+    """Fetch jobs for a specific run."""
+    url = f"https://api.github.com/repos/{repo}/actions/runs/{run_id}/jobs?per_page=100"
+    result = subprocess.run(
+        ["curl", "-sS", "-H", "Accept: application/vnd.github+json", url],
+        capture_output=True, text=True
+    )
+    return json.loads(result.stdout)
+
+
+def parse_duration(started, completed):
+    """Return duration in seconds between two ISO timestamps."""
+    if not started or not completed:
+        return 0
+    try:
+        s = datetime.fromisoformat(started.replace("Z", "+00:00"))
+        c = datetime.fromisoformat(completed.replace("Z", "+00:00"))
+        return max(0, (c - s).total_seconds())
+    except Exception:
+        return 0
+
+
+def main():
+    args = parse_args()
+    repo = args.repo
+    date_str = args.date
+    brief = args.mode == "brief"
+
+    print(f"Fetching workflow runs for {repo} on {date_str}...")
+    print("=" * 100)
+
+    all_runs = []
+    for page in range(1, 5):  # up to 400 runs
+        data = fetch_runs(repo, date_str, page=page)
+        runs = data.get("workflow_runs", [])
+        if not runs:
+            break
+        all_runs.extend(runs)
+        if len(runs) < 100:
+            break
+
+    print(f"Total workflow runs found: {len(all_runs)}")
+    print()
+
+    # Group by workflow name
+    workflow_stats = {}
+    for run in all_runs:
+        name = run.get("name", "Unknown")
+        event = run.get("event", "unknown")
+        conclusion = run.get("conclusion", "unknown")
+        run_id = run.get("id")
+
+        if name not in workflow_stats:
+            workflow_stats[name] = {
+                "count": 0,
+                "events": {},
+                "conclusions": {},
+                "total_job_seconds": 0,
+                "total_jobs": 0,
+                "run_ids": [],
+            }
+
+        workflow_stats[name]["count"] += 1
+        workflow_stats[name]["events"][event] = workflow_stats[name]["events"].get(event, 0) + 1
+        workflow_stats[name]["conclusions"][conclusion] = workflow_stats[name]["conclusions"].get(conclusion, 0) + 1
+        workflow_stats[name]["run_ids"].append(run_id)
+
+    # For each workflow, sample up to 3 runs to get job-level timing
+    print("Sampling job-level timing (up to 3 runs per workflow)...")
+    print()
+
+    for name, stats in workflow_stats.items():
+        sample_ids = stats["run_ids"][:3]
+        for run_id in sample_ids:
+            jobs_data = fetch_jobs(repo, run_id)
+            jobs = jobs_data.get("jobs", [])
+            for job in jobs:
+                started = job.get("started_at")
+                completed = job.get("completed_at")
+                duration = parse_duration(started, completed)
+                stats["total_job_seconds"] += duration
+                stats["total_jobs"] += 1
+
+        # Extrapolate: if we sampled N runs but there are M total, scale up
+        sampled = len(sample_ids)
+        total = stats["count"]
+        if sampled > 0 and sampled < total:
+            scale = total / sampled
+            stats["estimated_total_seconds"] = stats["total_job_seconds"] * scale
+        else:
+            stats["estimated_total_seconds"] = stats["total_job_seconds"]
+
+    # Print summary sorted by estimated cost (descending)
+    sorted_workflows = sorted(
+        workflow_stats.items(),
+        key=lambda x: x[1]["estimated_total_seconds"],
+        reverse=True
+    )
+
+    if brief:
+        # Brief mode: compact billable hours table
+        print(f"{'Workflow':<40} {'Runs':>5} {'Est.Mins':>9} {'Est.Hours':>10}")
+        print("-" * 68)
+        grand_total_minutes = 0
+        for name, stats in sorted_workflows:
+            est_mins = stats["estimated_total_seconds"] / 60
+            grand_total_minutes += est_mins
+            print(f"{name:<40} {stats['count']:>5} {est_mins:>9.1f} {est_mins/60:>10.2f}")
+        print("-" * 68)
+        print(f"{'TOTAL':<40} {len(all_runs):>5} {grand_total_minutes:>9.0f} {grand_total_minutes/60:>10.1f}")
+        print(f"\nProjected monthly: ~{grand_total_minutes/60*30:.0f} hours")
+    else:
+        # Full mode: detailed breakdown with per-run list
+        print("=" * 100)
+        print(f"{'Workflow':<40} {'Runs':>5} {'SampledJobs':>12} {'SampledMins':>12} {'Est.TotalMins':>14} {'Events'}")
+        print("-" * 100)
+
+        grand_total_minutes = 0
+        for name, stats in sorted_workflows:
+            sampled_mins = stats["total_job_seconds"] / 60
+            est_total_mins = stats["estimated_total_seconds"] / 60
+            grand_total_minutes += est_total_mins
+            events_str = ", ".join(f"{k}={v}" for k, v in stats["events"].items())
+            conclusions_str = ", ".join(f"{k}={v}" for k, v in stats["conclusions"].items())
+            print(
+                f"{name:<40} {stats['count']:>5} {stats['total_jobs']:>12} "
+                f"{sampled_mins:>12.1f} {est_total_mins:>14.1f}   {events_str}"
+            )
+            print(f"{'':>40} {'':>5} {'':>12} {'':>12} {'':>14}   outcomes: {conclusions_str}")
+
+        print("-" * 100)
+        print(f"{'GRAND TOTAL':>40} {len(all_runs):>5} {'':>12} {'':>12} {grand_total_minutes:>14.1f}")
+        print(f"\nEstimated total billable minutes on {date_str}: {grand_total_minutes:.0f} min ({grand_total_minutes/60:.1f} hours)")
+        print()
+
+        # Also show raw run list
+        print("\n" + "=" * 100)
+        print("DETAILED RUN LIST")
+        print("=" * 100)
+        for run in all_runs:
+            name = run.get("name", "Unknown")
+            event = run.get("event", "unknown")
+            conclusion = run.get("conclusion", "unknown")
+            run_id = run.get("id")
+            started = run.get("run_started_at", "?")
+            print(f"  [{run_id}] {name:<40} conclusion={conclusion:<12} event={event:<20} started={started}")
+
+
+if __name__ == "__main__":
+    main()