diff --git a/.github/workflows/dependency-check.yml b/.github/workflows/dependency-check.yml new file mode 100644 index 000000000..7ddd1ee04 --- /dev/null +++ b/.github/workflows/dependency-check.yml @@ -0,0 +1,211 @@ +name: Dependency Status Check + +on: + workflow_dispatch: + inputs: + check_type: + description: "Type of dependency check" + required: false + default: "all" + type: choice + options: + - all + - node + - dotnet + - docker + - npm + schedule: + - cron: "0 11 * * 1" # Weekly on Monday at 11 AM + +jobs: + dependency-status: + runs-on: ubuntu-latest + outputs: + node20-status: ${{ steps.check-versions.outputs.node20-status }} + node24-status: ${{ steps.check-versions.outputs.node24-status }} + dotnet-status: ${{ steps.check-versions.outputs.dotnet-status }} + docker-status: ${{ steps.check-versions.outputs.docker-status }} + buildx-status: ${{ steps.check-versions.outputs.buildx-status }} + npm-vulnerabilities: ${{ steps.check-versions.outputs.npm-vulnerabilities }} + open-dependency-prs: ${{ steps.check-prs.outputs.open-dependency-prs }} + steps: + - uses: actions/checkout@v5 + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Check dependency versions + id: check-versions + run: | + echo "## Dependency Status Report" >> $GITHUB_STEP_SUMMARY + echo "Generated on: $(date)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + # Check Node versions + if [[ "${{ github.event.inputs.check_type }}" == "all" || "${{ github.event.inputs.check_type }}" == "node" ]]; then + echo "### Node.js Versions" >> $GITHUB_STEP_SUMMARY + + VERSIONS_JSON=$(curl -s https://raw.githubusercontent.com/actions/node-versions/main/versions-manifest.json) + LATEST_NODE20=$(echo "$VERSIONS_JSON" | jq -r '.[] | select(.version | startswith("20.")) | .version' | head -1) + LATEST_NODE24=$(echo "$VERSIONS_JSON" | jq -r '.[] | select(.version | startswith("24.")) | .version' | head -1) + + CURRENT_NODE20=$(grep "NODE20_VERSION=" src/Misc/externals.sh | cut -d'"' -f2) + CURRENT_NODE24=$(grep "NODE24_VERSION=" src/Misc/externals.sh | cut -d'"' -f2) + + NODE20_STATUS="✅ up-to-date" + NODE24_STATUS="✅ up-to-date" + + if [ "$CURRENT_NODE20" != "$LATEST_NODE20" ]; then + NODE20_STATUS="⚠️ outdated" + fi + + if [ "$CURRENT_NODE24" != "$LATEST_NODE24" ]; then + NODE24_STATUS="⚠️ outdated" + fi + + echo "| Version | Current | Latest | Status |" >> $GITHUB_STEP_SUMMARY + echo "|---------|---------|--------|--------|" >> $GITHUB_STEP_SUMMARY + echo "| Node 20 | $CURRENT_NODE20 | $LATEST_NODE20 | $NODE20_STATUS |" >> $GITHUB_STEP_SUMMARY + echo "| Node 24 | $CURRENT_NODE24 | $LATEST_NODE24 | $NODE24_STATUS |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "node20-status=$NODE20_STATUS" >> $GITHUB_OUTPUT + echo "node24-status=$NODE24_STATUS" >> $GITHUB_OUTPUT + fi + + # Check .NET version + if [[ "${{ github.event.inputs.check_type }}" == "all" || "${{ github.event.inputs.check_type }}" == "dotnet" ]]; then + echo "### .NET SDK Version" >> $GITHUB_STEP_SUMMARY + + current_dotnet_version=$(jq -r .sdk.version ./src/global.json) + current_major_minor=$(echo "$current_dotnet_version" | cut -d '.' -f 1,2) + latest_dotnet_version=$(curl -sb -H "Accept: application/json" "https://dotnetcli.blob.core.windows.net/dotnet/Sdk/$current_major_minor/latest.version") + + DOTNET_STATUS="✅ up-to-date" + if [ "$current_dotnet_version" != "$latest_dotnet_version" ]; then + DOTNET_STATUS="⚠️ outdated" + fi + + echo "| Component | Current | Latest | Status |" >> $GITHUB_STEP_SUMMARY + echo "|-----------|---------|--------|--------|" >> $GITHUB_STEP_SUMMARY + echo "| .NET SDK | $current_dotnet_version | $latest_dotnet_version | $DOTNET_STATUS |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "dotnet-status=$DOTNET_STATUS" >> $GITHUB_OUTPUT + fi + + # Check Docker versions + if [[ "${{ github.event.inputs.check_type }}" == "all" || "${{ github.event.inputs.check_type }}" == "docker" ]]; then + echo "### Docker Versions" >> $GITHUB_STEP_SUMMARY + + current_docker=$(grep "ARG DOCKER_VERSION=" ./images/Dockerfile | cut -d'=' -f2) + current_buildx=$(grep "ARG BUILDX_VERSION=" ./images/Dockerfile | cut -d'=' -f2) + + latest_docker=$(curl -s https://download.docker.com/linux/static/stable/x86_64/ | grep -o 'docker-[0-9]*\.[0-9]*\.[0-9]*\.tgz' | sort -V | tail -n 1 | sed 's/docker-\(.*\)\.tgz/\1/') + latest_buildx=$(curl -s https://api.github.com/repos/docker/buildx/releases/latest | jq -r '.tag_name' | sed 's/^v//') + + DOCKER_STATUS="✅ up-to-date" + BUILDX_STATUS="✅ up-to-date" + + if [ "$current_docker" != "$latest_docker" ]; then + DOCKER_STATUS="⚠️ outdated" + fi + + if [ "$current_buildx" != "$latest_buildx" ]; then + BUILDX_STATUS="⚠️ outdated" + fi + + echo "| Component | Current | Latest | Status |" >> $GITHUB_STEP_SUMMARY + echo "|-----------|---------|--------|--------|" >> $GITHUB_STEP_SUMMARY + echo "| Docker | $current_docker | $latest_docker | $DOCKER_STATUS |" >> $GITHUB_STEP_SUMMARY + echo "| Docker Buildx | $current_buildx | $latest_buildx | $BUILDX_STATUS |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + + echo "docker-status=$DOCKER_STATUS" >> $GITHUB_OUTPUT + echo "buildx-status=$BUILDX_STATUS" >> $GITHUB_OUTPUT + fi + + # Check npm vulnerabilities + if [[ "${{ github.event.inputs.check_type }}" == "all" || "${{ github.event.inputs.check_type }}" == "npm" ]]; then + echo "### NPM Security Audit" >> $GITHUB_STEP_SUMMARY + + cd src/Misc/expressionFunc/hashFiles + npm install --silent + + AUDIT_OUTPUT="" + AUDIT_EXIT_CODE=0 + # Run npm audit and capture output and exit code + if ! AUDIT_OUTPUT=$(npm audit --json 2>&1); then + AUDIT_EXIT_CODE=$? + fi + + # Check if output is valid JSON + if echo "$AUDIT_OUTPUT" | jq . >/dev/null 2>&1; then + VULN_COUNT=$(echo "$AUDIT_OUTPUT" | jq '.metadata.vulnerabilities.total // 0') + # Ensure VULN_COUNT is a number + VULN_COUNT=$(echo "$VULN_COUNT" | grep -o '[0-9]*' | head -1) + VULN_COUNT=${VULN_COUNT:-0} + + NPM_STATUS="✅ no vulnerabilities" + if [ "$VULN_COUNT" -gt 0 ] 2>/dev/null; then + NPM_STATUS="⚠️ $VULN_COUNT vulnerabilities found" + + # Get vulnerability details + HIGH_VULNS=$(echo "$AUDIT_OUTPUT" | jq '.metadata.vulnerabilities.high // 0') + CRITICAL_VULNS=$(echo "$AUDIT_OUTPUT" | jq '.metadata.vulnerabilities.critical // 0') + + echo "| Severity | Count |" >> $GITHUB_STEP_SUMMARY + echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY + echo "| Critical | $CRITICAL_VULNS |" >> $GITHUB_STEP_SUMMARY + echo "| High | $HIGH_VULNS |" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + else + echo "No npm vulnerabilities found ✅" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + fi + else + NPM_STATUS="❌ npm audit failed" + echo "npm audit failed to run or returned invalid JSON ❌" >> $GITHUB_STEP_SUMMARY + echo "Exit code: $AUDIT_EXIT_CODE" >> $GITHUB_STEP_SUMMARY + echo "Output: $AUDIT_OUTPUT" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + fi + + echo "npm-vulnerabilities=$NPM_STATUS" >> $GITHUB_OUTPUT + fi + + - name: Check for open dependency PRs + id: check-prs + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + echo "### Open Dependency PRs" >> $GITHUB_STEP_SUMMARY + + # Get open PRs with dependency label + OPEN_PRS=$(gh pr list --label "dependencies" --state open --json number,title,url) + PR_COUNT=$(echo "$OPEN_PRS" | jq '. | length') + + if [ "$PR_COUNT" -gt 0 ]; then + echo "Found $PR_COUNT open dependency PR(s):" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "$OPEN_PRS" | jq -r '.[] | "- [#\(.number)](\(.url)) \(.title)"' >> $GITHUB_STEP_SUMMARY + else + echo "No open dependency PRs found ✅" >> $GITHUB_STEP_SUMMARY + fi + + echo "" >> $GITHUB_STEP_SUMMARY + echo "open-dependency-prs=$PR_COUNT" >> $GITHUB_OUTPUT + + - name: Summary + run: | + echo "### Summary" >> $GITHUB_STEP_SUMMARY + echo "- Check for open PRs with the \`dependency\` label before releases" >> $GITHUB_STEP_SUMMARY + echo "- Review and merge dependency updates regularly" >> $GITHUB_STEP_SUMMARY + echo "- Critical vulnerabilities should be addressed immediately" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Automated workflows run weekly to check for updates:**" >> $GITHUB_STEP_SUMMARY + echo "- Node.js versions (Mondays at 6 AM)" >> $GITHUB_STEP_SUMMARY + echo "- NPM audit fix (Mondays at 7 AM)" >> $GITHUB_STEP_SUMMARY + echo "- .NET SDK updates (Mondays at midnight)" >> $GITHUB_STEP_SUMMARY + echo "- Docker/Buildx updates (Mondays at midnight)" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/docker-buildx-upgrade.yml b/.github/workflows/docker-buildx-upgrade.yml index c97d10009..021ed649f 100644 --- a/.github/workflows/docker-buildx-upgrade.yml +++ b/.github/workflows/docker-buildx-upgrade.yml @@ -2,7 +2,7 @@ name: "Docker/Buildx Version Upgrade" on: schedule: - - cron: "0 9 * * 1" # Weekly on Monday at 9 AM UTC (independent of other dependencies) + - cron: "0 0 * * 1" # Run every Monday at midnight workflow_dispatch: # Allow manual triggering jobs: @@ -159,5 +159,8 @@ jobs: # Create PR gh pr create -B main -H "$branch_name" \ --title "$pr_title" \ - --label "dependency" \ + --label "dependencies" \ + --label "dependencies-weekly-check" \ + --label "dependencies-not-dependabot" \ + --label "docker" \ --body-file pr_body.txt diff --git a/.github/workflows/dotnet-upgrade.yml b/.github/workflows/dotnet-upgrade.yml index 9f727a7d8..43ccc4040 100644 --- a/.github/workflows/dotnet-upgrade.yml +++ b/.github/workflows/dotnet-upgrade.yml @@ -96,7 +96,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - gh pr create -B main -H feature/dotnetsdk-upgrade/${{ needs.dotnet-update.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }} --title "Update dotnet sdk to latest version @${{ needs.dotnet-update.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}" --label "dependency" --body " + gh pr create -B main -H feature/dotnetsdk-upgrade/${{ needs.dotnet-update.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }} --title "Update dotnet sdk to latest version @${{ needs.dotnet-update.outputs.DOTNET_LATEST_MAJOR_MINOR_PATCH_VERSION }}" --label "dependencies" --label "dependencies-weekly-check" --label "dependencies-not-dependabot" --label "dotnet" --body " https://dotnetcli.blob.core.windows.net/dotnet/Sdk/${{ needs.dotnet-update.outputs.DOTNET_CURRENT_MAJOR_MINOR_VERSION }}/latest.version diff --git a/.github/workflows/node-upgrade.yml b/.github/workflows/node-upgrade.yml index 3aeff98c9..a8db2b115 100644 --- a/.github/workflows/node-upgrade.yml +++ b/.github/workflows/node-upgrade.yml @@ -120,7 +120,11 @@ jobs: # Create PR gh pr create -B main -H "$branch_name" \ --title "chore: update Node versions" \ - --label "dependency" \ + --label "dependencies" \ + --label "dependencies-weekly-check" \ + --label "dependencies-not-dependabot" \ + --label "node" \ + --label "javascript" \ --body-file pr_body.txt echo "::notice title=PR Created::Successfully created Node.js version update PR on branch $branch_name" diff --git a/.github/workflows/npm-audit-typescript.yml b/.github/workflows/npm-audit-typescript.yml index 1b90db71b..ceac1b5c3 100644 --- a/.github/workflows/npm-audit-typescript.yml +++ b/.github/workflows/npm-audit-typescript.yml @@ -220,9 +220,9 @@ jobs: fi # Create PR with appropriate labels - labels="dependency,typescript" + labels="dependencies,dependencies-not-dependabot,typescript,npm,security" if [[ "$build_status" == *"fails"* ]]; then - labels="dependency,typescript,needs-manual-review" + labels="dependencies,dependencies-not-dependabot,typescript,npm,security,needs-manual-review" fi # Create PR diff --git a/.github/workflows/npm-audit.yml b/.github/workflows/npm-audit.yml index 2372a07c6..73d6aed32 100644 --- a/.github/workflows/npm-audit.yml +++ b/.github/workflows/npm-audit.yml @@ -125,7 +125,12 @@ jobs: # Create PR gh pr create -B main -H "$branch_name" \ --title "chore: npm audit fix for hashFiles dependencies" \ - --label "dependency" \ + --label "dependencies" \ + --label "dependencies-weekly-check" \ + --label "dependencies-not-dependabot" \ + --label "npm" \ + --label "typescript" \ + --label "security" \ --body-file pr_body.txt else echo "✅ No changes to commit - npm audit fix did not modify any files" diff --git a/docs/dependency-management.md b/docs/dependency-management.md new file mode 100644 index 000000000..9f20540c0 --- /dev/null +++ b/docs/dependency-management.md @@ -0,0 +1,217 @@ +# Runner Dependency Management Process + +## Overview + +This document outlines the automated dependency management process for the GitHub Actions Runner, designed to ensure we maintain up-to-date and secure dependencies while providing predictable release cycles. + +## Release Schedule + +- **Monthly Runner Releases**: New runner versions are released monthly +- **Weekly Dependency Checks**: Automated workflows check for dependency updates every Monday +- **Security Patches**: Critical security vulnerabilities are addressed immediately outside the regular schedule + +## Automated Workflows + +**Note**: These workflows are implemented across separate PRs for easier review and independent deployment. Each workflow includes comprehensive error handling and security-focused vulnerability detection. + +### 1. Foundation Labels + +- **Workflow**: `.github/workflows/setup-labels.yml` (PR #4024) +- **Purpose**: Creates consistent dependency labels for all automation workflows +- **Labels**: `dependencies`, `security`, `typescript`, `needs-manual-review` +- **Prerequisite**: Must be merged before other workflows for proper labeling + +### 2. Node.js Version Updates + +- **Workflow**: `.github/workflows/node-upgrade.yml` +- **Schedule**: Mondays at 6:00 AM UTC +- **Purpose**: Updates Node.js 20 and 24 versions in `src/Misc/externals.sh` +- **Source**: [nodejs.org](https://nodejs.org) and [actions/alpine_nodejs](https://github.com/actions/alpine_nodejs) +- **Priority**: First (NPM depends on current Node.js versions) + +### 3. NPM Security Audit + +- **Primary Workflow**: `.github/workflows/npm-audit.yml` ("NPM Audit Fix") + - **Schedule**: Mondays at 7:00 AM UTC + - **Purpose**: Automated security vulnerability detection and basic fixes + - **Location**: `src/Misc/expressionFunc/hashFiles/` + - **Features**: npm audit, security patch application, PR creation + - **Dependency**: Runs after Node.js updates for optimal compatibility + +- **Fallback Workflow**: `.github/workflows/npm-audit-typescript.yml` ("NPM Audit Fix with TypeScript Auto-Fix") + - **Trigger**: Manual dispatch only + - **Purpose**: Manual security audit with TypeScript compatibility fixes + - **Use Case**: When scheduled workflow fails or needs custom intervention + - **Features**: Enhanced TypeScript auto-repair, graduated security response + - **How to Use**: + 1. If the scheduled "NPM Audit Fix" workflow fails, go to Actions tab + 2. Select "NPM Audit Fix with TypeScript Auto-Fix" workflow + 3. Click "Run workflow" and optionally specify fix level (auto/manual) + 4. Review the generated PR for TypeScript compatibility issues + +### 4. .NET SDK Updates + +- **Workflow**: `.github/workflows/dotnet-upgrade.yml` +- **Schedule**: Mondays at midnight UTC +- **Purpose**: Updates .NET SDK and package versions with build validation +- **Features**: Global.json updates, NuGet package management, compatibility checking +- **Independence**: Runs independently of Node.js/NPM updates + +### 5. Docker/Buildx Updates + +- **Workflow**: `.github/workflows/docker-buildx-upgrade.yml` ("Docker/Buildx Version Upgrade") +- **Schedule**: Mondays at midnight UTC +- **Purpose**: Updates Docker and Docker Buildx versions with multi-platform validation +- **Features**: Container security scanning, multi-architecture build testing +- **Independence**: Runs independently of other dependency updates + +### 6. Dependency Monitoring + +- **Workflow**: `.github/workflows/dependency-check.yml` ("Dependency Status Check") +- **Schedule**: Mondays at 11:00 AM UTC +- **Purpose**: Comprehensive status report of all dependencies with security audit +- **Features**: Multi-dependency checking, npm audit status, build validation, choice of specific component checks +- **Summary**: Runs last to capture results from all morning dependency updates + +## Release Process Integration + +### Pre-Release Checklist + +Before each monthly runner release: + +1. **Check Dependency PRs**: + + ```bash + # List all open dependency PRs + gh pr list --label "dependencies" --state open + + # List only automated weekly dependency updates + gh pr list --label "dependencies-weekly-check" --state open + + # List only custom dependency automation (not dependabot) + gh pr list --label "dependencies-not-dependabot" --state open + ``` + +2. **Run Manual Dependency Check**: + - Go to Actions tab → "Dependency Status Check" → "Run workflow" + - Review the summary for any outdated dependencies + +3. **Review and Merge Updates**: + - Prioritize security-related updates + - Test dependency updates in development environment + - Merge approved dependency PRs + +### Vulnerability Response + +#### Critical Security Vulnerabilities + +- **Response Time**: Within 24 hours +- **Process**: + 1. Assess impact on runner security + 2. Create hotfix branch if runner data security is affected + 3. Expedite patch release if necessary + 4. Document in security advisory if applicable + +#### Non-Critical Vulnerabilities + +- **Response Time**: Next monthly release +- **Process**: + 1. Evaluate if vulnerability affects runner functionality + 2. Include fix in regular dependency update cycle + 3. Document in release notes + +## Monitoring and Alerts + +### GitHub Actions Workflow Status + +- All dependency workflows create PRs with the `dependencies` label +- Failed workflows should be investigated immediately +- Weekly dependency status reports are generated automatically + +### Manual Checks + +You can manually trigger dependency checks: + +- **Full Status**: Run "Dependency Status Check" workflow +- **Specific Component**: Use the dropdown to check individual dependencies + +## Dependency Labels + +All automated dependency PRs are tagged with labels for easy filtering and management: + +### Primary Labels + +- **`dependencies`**: All automated dependency-related PRs +- **`dependencies-weekly-check`**: Automated weekly dependency updates from scheduled workflows +- **`dependencies-not-dependabot`**: Custom dependency automation (not created by dependabot) +- **`security`**: Security vulnerability fixes and patches +- **`typescript`**: TypeScript compatibility and type definition updates +- **`needs-manual-review`**: Complex updates requiring human verification + +### Technology-Specific Labels + +- **`node`**: Node.js version updates +- **`javascript`**: JavaScript runtime and tooling updates +- **`npm`**: NPM package and security updates +- **`dotnet`**: .NET SDK and NuGet package updates +- **`docker`**: Docker and container tooling updates + +### Workflow-Specific Branches + +- **Node.js updates**: `chore/update-node` branch +- **NPM security fixes**: `chore/npm-audit-fix-YYYYMMDD` and `chore/npm-audit-fix-with-ts-repair` branches +- **NuGet/.NET updates**: `feature/dotnetsdk-upgrade/{version}` branches +- **Docker updates**: `feature/docker-buildx-upgrade` branch + +## Special Considerations + +### Node.js Updates + +When updating Node.js versions, remember to: + +1. Create a corresponding release in [actions/alpine_nodejs](https://github.com/actions/alpine_nodejs) +2. Follow the alpine_nodejs getting started guide +3. Test container builds with new Node versions + +### .NET SDK Updates + +- Only patch versions are auto-updated within the same major.minor version +- Major/minor version updates require manual review and testing + +### Docker Updates + +- Updates include both Docker Engine and Docker Buildx +- Verify compatibility with runner container workflows + +## Troubleshooting + +### Common Issues + +1. **NPM Audit Workflow Fails**: + - Check if `package.json` exists in `src/Misc/expressionFunc/hashFiles/` + - Verify Node.js setup step succeeded + +2. **Version Detection Fails**: + - Check if upstream APIs are available + - Verify parsing logic for version extraction + +3. **PR Creation Fails**: + - Ensure `GITHUB_TOKEN` has sufficient permissions + - Check if branch already exists + +### Contact + +For questions about the dependency management process: + +- Create an issue with the `dependencies` label +- Review existing dependency management workflows +- Consult the runner team for security-related concerns + +## Metrics and KPIs + +Track these metrics to measure dependency management effectiveness: + +- Number of open dependency PRs at release time +- Time to merge dependency updates +- Number of security vulnerabilities by severity +- Release cycle adherence (monthly target)