Feature/ch 125 GitHub actions skaffold

.gitignore

@@ -10,6 +10,8 @@ node_modules
 *.DS_Store
 deployment/helm
 deployment/compose
+deployment/skaffold-ci.yaml
+deployment/skaffold-images.json
 /docker-compose.yaml
 *.egg-info
 *.idea

CLAUDE.md

@@ -58,8 +58,8 @@ Always activate `conda activate ch` before any backend command.
 
 ## Key Scripts
 
-- `harness-generate` — generate code
-- `harness-deployment` — generate deployment files (Helm charts, CI/CD files, etc.)
+- `harness-generate` — generate code; `harness-generate ci-cd` generates the CI/CD pipelines
+- `harness-deployment` — generate deployment files (Helm charts, Skaffold, etc.)
 - `harness-application` — generate application code (e.g. Django apps)
 - `harness-migrate` — migration helper
 - `ch_cli_tools` — Python package for deployment and other tools

deployment-configuration/README.md

@@ -10,8 +10,14 @@ Those files are used by the script `harness-deployment` script.
   custom `values.yaml` to your application over changing this file.
 - `values-template-[env].yaml` / `value-template-[env].yaml`: environment specific overrides of the above, loaded
   when the environment is selected with `harness-deployment -e [env]`. See [environments](../docs/build-deploy/environments.md)
-- `codefresh-template-[dev|prod].yaml`: base for `codefresh/codefresh-[dev|prod].yaml`. Modify this file if you want to
-  change the build steps inside codefresh
+- `codefresh-template-[dev|prod].yaml`: base for `deployment/ci--[NAME]--[dev|prod].yaml`, generated with
+  `harness-generate ci-cd`. Modify this file if you want to change the build steps inside codefresh
 - `codefresh-build-template.yaml`: base for a single build entry in `codefresh.yaml`
 - `codefresh-publish-template.yaml`: base for a single publish (image tagging) entry in `codefresh.yaml`
+- `github-template.yaml`: base for the GitHub Actions workflows (`.github/workflows/ci--[NAME]--[env].yaml`),
+  generated with `harness-generate ci-cd`: the job prototypes expanded per image/app and the shared steps
+- `github-template-[dev|stage|prod|test].yaml`: per-environment workflow name, triggers and deltas, merged over
+  the base template. Override any part from the same files in your project's `deployment-configuration/`: a
+  project gets a pipeline only for the environments it has a `codefresh-template-[env].yaml` or
+  `github-template-[env].yaml` for (an empty file inherits everything)
 

deployment-configuration/codefresh-template-prod.yaml

@@ -34,7 +34,7 @@ steps:
       - export HELM_CHART_VERSION_ARG="$( [ -n "${{CHART_VERSION}}" ] && printf -- "--chart-version %s" "${{CHART_VERSION}}" )"
       - export HELM_APP_VERSION_ARG="$( APP_VERSION_RESOLVED="${{APP_VERSION}}"; [ -z "$APP_VERSION_RESOLVED" ] && APP_VERSION_RESOLVED="${{DEPLOYMENT_TAG}}"; [ -n "$APP_VERSION_RESOLVED" ] && printf -- "--app-version %s" "$APP_VERSION_RESOLVED" )"
       - export HELM_META_ARGS="$HELM_NAME_ARG $HELM_CHART_VERSION_ARG $HELM_APP_VERSION_ARG"
-      - harness-deployment $PATHS -t ${{DEPLOYMENT_TAG}} -d ${{DOMAIN}} -r ${{REGISTRY}} -rsn '${{REGISTRY_SECRET}}' -n ${{NAMESPACE}} -e $ENV --no-cd $HELM_META_ARGS $PARAMS
+      - harness-deployment $PATHS -t ${{DEPLOYMENT_TAG}} -d ${{DOMAIN}} -r ${{REGISTRY}} -rsn '${{REGISTRY_SECRET}}' -n ${{NAMESPACE}} -e $ENV $HELM_META_ARGS $PARAMS
   prepare_deployment_view:
     commands:
       - "helm template ./deployment/helm --debug -n ${{NAMESPACE}}"

deployment-configuration/codefresh-template-stage.yaml

@@ -37,7 +37,7 @@ steps:
       - export HELM_CHART_VERSION_ARG="$( [ -n "${{CHART_VERSION}}" ] && printf -- "--chart-version %s" "${{CHART_VERSION}}" )"
       - export HELM_APP_VERSION_ARG="$( APP_VERSION_RESOLVED="${{APP_VERSION}}"; [ -z "$APP_VERSION_RESOLVED" ] && APP_VERSION_RESOLVED="${{DEPLOYMENT_TAG}}"; [ -n "$APP_VERSION_RESOLVED" ] && printf -- "--app-version %s" "$APP_VERSION_RESOLVED" )"
       - export HELM_META_ARGS="$HELM_NAME_ARG $HELM_CHART_VERSION_ARG $HELM_APP_VERSION_ARG"
-      - harness-deployment $PATHS -t ${{DEPLOYMENT_TAG}} -d ${{DOMAIN}} -r ${{REGISTRY}} -rsn ${{REGISTRY_SECRET}} -n ${{NAMESPACE}} -e $ENV --no-cd $HELM_META_ARGS $PARAMS
+      - harness-deployment $PATHS -t ${{DEPLOYMENT_TAG}} -d ${{DOMAIN}} -r ${{REGISTRY}} -rsn ${{REGISTRY_SECRET}} -n ${{NAMESPACE}} -e $ENV $HELM_META_ARGS $PARAMS
   prepare_deployment_view:
     commands:
       - "helm template ./deployment/helm --debug -n ${{NAMESPACE}}"

deployment-configuration/codefresh-template-test.yaml

@@ -37,7 +37,7 @@ steps:
       - export HELM_CHART_VERSION_ARG="$( [ -n "${{CHART_VERSION}}" ] && printf -- "--chart-version %s" "${{CHART_VERSION}}" )"
       - export HELM_APP_VERSION_ARG="$( APP_VERSION_RESOLVED="${{APP_VERSION}}"; [ -z "$APP_VERSION_RESOLVED" ] && APP_VERSION_RESOLVED="${{DEPLOYMENT_TAG}}"; [ -n "$APP_VERSION_RESOLVED" ] && printf -- "--app-version %s" "$APP_VERSION_RESOLVED" )"
       - export HELM_META_ARGS="$HELM_NAME_ARG $HELM_CHART_VERSION_ARG $HELM_APP_VERSION_ARG"
-      - harness-deployment $PATHS -n test-${{NAMESPACE_BASENAME}} -d ${{DOMAIN}} -r ${{REGISTRY}} -rsn ${{REGISTRY_SECRET}} -e $ENV --write-env --cache-url '${{IMAGE_CACHE_URL}}' -N $HELM_META_ARGS $PARAMS
+      - harness-deployment $PATHS -n test-${{NAMESPACE_BASENAME}} -d ${{DOMAIN}} -r ${{REGISTRY}} -rsn ${{REGISTRY_SECRET}} -e $ENV --write-env --cache-url '${{IMAGE_CACHE_URL}}' $HELM_META_ARGS $PARAMS
       - cat deployment/.env >> ${{CF_VOLUME_PATH}}/env_vars_to_export
       - cat ${{CF_VOLUME_PATH}}/env_vars_to_export
   prepare_deployment_view:

deployment-configuration/github-template-dev.yaml

@@ -0,0 +1,104 @@
+# Development pipeline: build, unit tests, deployment and API/E2E tests. Its runs are promoted with the
+# workflow in `cloudharness.promote`, written next to this one as cloudharness-promote.yaml.
+name: CloudHarness $ENV
+on:
+  push:
+    branches: [develop]
+  workflow_dispatch:
+    inputs:
+      force_build:
+        description: Comma-separated image names to rebuild even if their tag already exists, or `all`
+        type: string
+        default: ""
+      skip_tests:
+        description: Set to `true` to skip the tests
+        type: string
+        default: ""
+cloudharness:
+  # Promotes the images of one run of this workflow, started by hand after approval (protect the `publish`
+  # environment with required reviewers): every image the run deployed - deployment/skaffold-images.json in
+  # its cloudharness-deployment artifact - gets the given tag (and `latest` where it is published), then the
+  # run's commit gets the git tag, which starts the staging pipeline. A tag pushed with the workflow's own
+  # token starts no workflow: set the GH_PAT secret (contents: write) for that.
+  promote:
+    name: CloudHarness promote
+    on:
+      workflow_dispatch:
+        inputs:
+          tag:
+            description: Tag to give the images and the repository
+            type: string
+            required: true
+          run_id:
+            description: Run of $WORKFLOW whose images to promote (default its latest successful run on the default branch)
+            type: string
+            default: ""
+    permissions:
+      contents: write
+      actions: read
+    jobs:
+      promote:
+        name: Promote the images
+        environment: publish
+        env:
+          TAG: ${{ inputs.tag }}
+          REGISTRY: ${{ vars.REGISTRY }}
+          REGISTRY_PUBLISH_URL: ${{ vars.REGISTRY_PUBLISH_URL }}
+        steps:
+          - name: Find the run to promote
+            id: run
+            env:
+              GH_TOKEN: ${{ github.token }}
+              RUN_ID: ${{ inputs.run_id }}
+              DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+            run: |
+              if [ -z "$RUN_ID" ]; then
+                RUN_ID=$(gh run list --repo "$GITHUB_REPOSITORY" --workflow $WORKFLOW --branch "$DEFAULT_BRANCH" --status success --limit 1 --json databaseId --jq '.[0].databaseId')
+              fi
+              if [ -z "$RUN_ID" ]; then
+                echo "::error::No successful run of $WORKFLOW to promote"
+                exit 1
+              fi
+              sha=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$RUN_ID" --jq .head_sha)
+              echo "id=$RUN_ID" >> "$GITHUB_OUTPUT"
+              echo "sha=$sha" >> "$GITHUB_OUTPUT"
+              echo "Promoting run $RUN_ID (commit $sha) as $TAG" >> "$GITHUB_STEP_SUMMARY"
+          - name: Download the images of the run
+            uses: actions/download-artifact@v4
+            with:
+              name: cloudharness-deployment
+              path: deployment
+              run-id: ${{ steps.run.outputs.id }}
+              github-token: ${{ github.token }}
+          - uses: docker/login-action@v3
+            with:
+              registry: ${{ vars.REGISTRY }}
+              username: ${{ secrets.REGISTRY_USERNAME }}
+              password: ${{ secrets.REGISTRY_PASSWORD }}
+          - name: Log in to the publish registry
+            if: ${{ vars.REGISTRY_PUBLISH_URL != '' }}
+            env:
+              USERNAME: ${{ secrets.REGISTRY_PUBLISH_USERNAME || secrets.REGISTRY_USERNAME }}
+              PASSWORD: ${{ secrets.REGISTRY_PUBLISH_PASSWORD || secrets.REGISTRY_PASSWORD }}
+            run: echo "$PASSWORD" | docker login "${REGISTRY_PUBLISH_URL%%/*}" -u "$USERNAME" --password-stdin
+          # each image keeps its content-hash tag and gets the new one in the registry the pipelines deploy
+          # from; the publish registry (REGISTRY_PUBLISH_URL, default the same) also gets `latest`
+          - name: Tag the images
+            run: |
+              set -e
+              target="${REGISTRY_PUBLISH_URL:-$REGISTRY}"
+              jq -r '.builds[] | "\(.imageName) \(.tag)"' deployment/skaffold-images.json | while read -r image source; do
+                published="$target/${image#"$REGISTRY"/}"
+                tags="-t $image:$TAG -t $published:latest"
+                if [ "$published" != "$image" ]; then tags="$tags -t $published:$TAG"; fi
+                echo "$source -> $tags"
+                docker buildx imagetools create $tags "$source"
+              done
+          - uses: actions/checkout@v4
+            with:
+              ref: ${{ steps.run.outputs.sha }}
+              token: ${{ secrets.GH_PAT || github.token }}
+          - name: Tag the repository
+            run: |
+              git tag "$TAG"
+              git push origin "refs/tags/$TAG"

deployment-configuration/github-template-prod.yaml

@@ -0,0 +1,18 @@
+# Production pipeline: deploys the images promoted with the given tag, nothing else.
+# Protect the `prod` environment with required reviewers to get an approval step.
+name: CloudHarness $ENV
+on:
+  workflow_dispatch:
+    inputs:
+      deployment_tag:
+        description: Tag of the images to deploy (a tag given by the promote workflow)
+        type: string
+        required: true
+# deploys the promoted images: builds nothing
+cloudharness:
+  build: null
+jobs:
+  prepare:
+    name: Prepare helm chart
+  tests-api: null
+  tests-e2e: null

deployment-configuration/github-template-stage.yaml

@@ -0,0 +1,22 @@
+# Staging pipeline: started by a git tag (the promote workflow pushes one), deploys the images with that tag
+# and runs the API/E2E tests. It can also be started by hand for a tag.
+name: CloudHarness $ENV
+on:
+  push:
+    tags: ["*"]
+  workflow_dispatch:
+    inputs:
+      deployment_tag:
+        description: Tag of the images to deploy (a tag given by the promote workflow)
+        type: string
+        required: true
+      skip_tests:
+        description: Set to `true` to skip the tests
+        type: string
+        default: ""
+# deploys the promoted images: builds nothing
+cloudharness:
+  build: null
+jobs:
+  prepare:
+    name: Prepare helm chart

deployment-configuration/github-template-test.yaml

@@ -0,0 +1,43 @@
+# Testing pipeline: deploys the current code in its own namespace, runs every test and removes the deployment.
+name: CloudHarness $ENV
+on:
+  pull_request:
+    branches: [develop]
+  workflow_dispatch:
+    inputs:
+      force_build:
+        description: Comma-separated image names to rebuild even if their tag already exists, or `all`
+        type: string
+        default: ""
+      skip_tests:
+        description: Set to `true` to skip the tests
+        type: string
+        default: ""
+jobs:
+  prepare:
+    env:
+      NAMESPACE: test-${{ vars.NAMESPACE_BASENAME }}
+  # Deleting the deployment waits for approval: protect the `cleanup` environment with required reviewers
+  # (no deployment is recorded for it). The `test` environment needs none, so the deployment does not wait.
+  cleanup:
+    name: Delete the test deployment
+    needs: [prepare, deploy, tests-api, tests-e2e]
+    if: ${{ always() && needs.deploy.result == 'success' }}
+    environment:
+      name: cleanup
+      deployment: false
+    env:
+      NAMESPACE: ${{ needs.prepare.outputs.namespace }}
+      WAIT_ON_FAIL: ${{ vars.WAIT_ON_FAIL || '0' }}
+      FAILED: ${{ contains(needs.*.result, 'failure') }}
+    steps:
+      - uses: azure/setup-helm@v4
+      - $KUBE_AUTH
+      - name: Uninstall the release and delete the namespace
+        run: |
+          if [ "$FAILED" = "true" ] && [ "$WAIT_ON_FAIL" != "0" ]; then
+            echo "Tests failed: keeping the deployment for $WAIT_ON_FAIL minutes for inspection"
+            sleep $((WAIT_ON_FAIL * 60))
+          fi
+          helm uninstall "$NAMESPACE" -n "$NAMESPACE" || true
+          kubectl delete namespace "$NAMESPACE" --ignore-not-found