Add system level testing which can be run locally and in C-ACI

.github/workflows/ci.yml

@@ -1,4 +1,8 @@
-name: "KMS CI"
+name: CI
+
+permissions:
+  id-token: write
+  contents: read
 
 on:
   push:
@@ -8,46 +12,51 @@ on:
   workflow_dispatch:
 
 jobs:
-  kms:
-    name: kms
+  lint:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          persist-credentials: false
+      - name: Checkout
+        uses: actions/checkout@v4
 
-      - name: Build DevContainer
-        uses: devcontainers/[email protected]
-        with:
-          push: never
-          configFile: .devcontainer/devcontainer.json
-          runCmd: |
-            scripts/set_python_env.sh && pip install -r requirements.txt && make demo && npm run test
-        env:
-          KMS_WORKSPACE: ./workspace
+      - name: Run Lint
+        run: make lint
 
-  test-mccf-kms:
-    runs-on: ubuntu-20.04
+  unit:
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: Test mCCF KMS
-        run: |
-          . ./scripts/setup_mCCF.sh
-          ./scripts/test_curl.sh
+      - name: Install Dependencies
         env:
-          PUBLIC_CERT_PEM: ${{ secrets.PUBLIC_CERT_PEM }}
-          PRIVATE_CERT_PEM: ${{ secrets.PRIVATE_CERT_PEM }}
-          CCF_NAME: ${{ secrets.DEPLOYMENT_MCCF }}
+          GH_TOKEN: ${{ github.token }}
+        run: ./scripts/tools/install-deps.sh
 
-  lint:
-    runs-on: ubuntu-20.04
+      - name: Run Unit Tests
+        run: npm run test
+
+  e2e:
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4
 
-      - name: KMS Lint
-        run: make lint
+      - name: Install Dependencies
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: ./scripts/tools/install-deps.sh
+
+      - name: Build DevContainer
+        uses: devcontainers/[email protected]
+        env:
+          GH_TOKEN: ${{ github.token }}
+        with:
+          push: never
+          configFile: .devcontainer/devcontainer.json
+          runCmd: |
+             scripts/set_python_env.sh && pip install -r requirements.txt
+             make demo
+
+  system:
+    secrets: inherit # pragma: allowlist secret
+    uses: ./.github/workflows/system-tests.yml

.github/workflows/endpoint-test.yml

@@ -0,0 +1,50 @@
+name: Endpoint Test
+
+permissions:
+  id-token: write
+  contents: read
+
+on:
+  workflow_dispatch:
+    inputs:
+      endpoint:
+        type: string
+  workflow_call:
+    inputs:
+      endpoint:
+        type: string
+
+jobs:
+
+  endpoint:
+    name: ${{ inputs.endpoint }} (${{ matrix.env }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        env: [local]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Log into Azure
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.MANAGED_ID_CLIENT_ID }}
+          tenant-id: ${{ secrets.MANAGED_ID_TENANT_ID }}
+          subscription-id: ${{ vars.SUBSCRIPTION }}
+
+      - name: Install Dependencies
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: ./scripts/tools/install-deps.sh
+
+      - name: Run System Tests
+        env:
+            TEST_ENVIRONMENT: ${{ matrix.env }}
+        run: |
+          if [[ "$TEST_ENVIRONMENT" == "cloud" ]]; then
+            pytest -sv -n auto test/system-test/test_${{ inputs.endpoint }}.py
+          else
+            pytest -sv test/system-test/test_${{ inputs.endpoint }}.py
+          fi

.github/workflows/system-tests.yml

@@ -0,0 +1,36 @@
+name: System Tests
+
+permissions:
+  id-token: write
+  contents: read
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  discover-tests:
+    runs-on: ubuntu-latest
+    outputs:
+      endpoints: ${{ steps.find_tests.outputs.tests }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Find Test Files
+        id: find_tests
+        run: |
+          TEST_FILES=$(cd test/system-test && find . -name 'test_*.py' | sed -e 's|^\./||' -e 's|^test_||' -e 's|\.py$||')
+          JSON_ARRAY=$(printf '%s\n' "${TEST_FILES[@]}" | jq -R . | jq -s .)
+          echo "tests=$JSON_ARRAY" | sed ':a;N;$!ba;s/\n//g' >> $GITHUB_OUTPUT
+
+  endpoint:
+    needs: discover-tests
+    secrets: inherit # pragma: allowlist secret
+    strategy:
+      fail-fast: false
+      matrix:
+        endpoint: ${{ fromJson(needs.discover-tests.outputs.endpoints) }}
+    uses: ./.github/workflows/endpoint-test.yml
+    with:
+      endpoint: ${{ matrix.endpoint }}

.gitignore

@@ -3,10 +3,11 @@ node_modules
 package-lock.json
 vol
 .venv_ccf_sandbox
-workspace
+workspace*
 .env
 hello/
 ccf-app/
 nohup.out
 aad.env
-*.log
+*.log
+__pycache__/

Makefile

@@ -7,10 +7,14 @@ KEYS_DIR ?= ${KMS_WORKSPACE}/sandbox_common
 RUN_BACK ?= true
 CCF_PLATFORM ?= virtual
 
+DEPLOYMENT_ENV ?= $(if $(shell echo $(KMS_URL) | grep -E '127.0.0.1|localhost'),local,cloud)
+
+ifndef MEMBER_COUNT
 ifeq ($(findstring https://127.0.0.1,$(KMS_URL)),https://127.0.0.1)
-    MEMBER_COUNT := 3
+	MEMBER_COUNT := 3
 else
-    MEMBER_COUNT := 1
+	MEMBER_COUNT := 1
+endif
 endif
 
 CCF_SANDBOX_EXTRA_ARGS ?=
@@ -107,7 +111,7 @@ propose-key-rotation-policy: ## 🚀 Deploy the key rotation policy to the sandb
 refresh-key: ## 🚀 Refresh a key on the instance
 	@echo -e "\e[34m$@\e[0m" || true
 	$(call check_defined, KMS_URL)
-	@CCF_PLATFORM=${CCF_PLATFORM} sleep 20;curl "${KMS_URL}"/app/refresh -X POST --cacert "${KEYS_DIR}"/service_cert.pem  -H "Content-Type: application/json" -i  -w '\n'
+	@CCF_PLATFORM=${CCF_PLATFORM};curl "${KMS_URL}"/app/refresh -X POST --cacert "${KEYS_DIR}"/service_cert.pem  -H "Content-Type: application/json" -i  -w '\n'
 
 set-constitution: start-host-idp ## Set new custom constitution
 	@echo -e "\e[34m$@\e[0m" || true
@@ -138,6 +142,66 @@ lint: ## 🔍 Lint the code base (but don't fix)
 	@echo -e "\e[34m$@\e[0m" || true
 	@CCF_PLATFORM=${CCF_PLATFORM} ./scripts/lint.sh --fix
 
+# Manage Infra -----------------------------------------------------------------
+
+ccf-sandbox-up:
+	@WORKSPACE=${KMS_WORKSPACE} \
+	DEPLOYMENT_ENV=${DEPLOYMENT_ENV} \
+	IMAGE_TAG=${IMAGE_TAG} \
+	DEPLOYMENT_NAME=$(deployment-name) \
+		./scripts/ccf-sandbox-up.sh
+
+ccf-sandbox-down:
+	@DEPLOYMENT_ENV=${DEPLOYMENT_ENV} \
+	DEPLOYMENT_NAME=$(deployment-name) \
+		./scripts/ccf-sandbox-down.sh
+
+ccf-sandbox-attach:
+	@DEPLOYMENT_ENV=${DEPLOYMENT_ENV} \
+		./scripts/ccf-sandbox-attach.sh
+
+ccf-sandbox-logs:
+	@DEPLOYMENT_ENV=${DEPLOYMENT_ENV} \
+		./scripts/ccf-sandbox-logs.sh
+
+jwt-issuer-up:
+	@WORKSPACE=${KMS_WORKSPACE} \
+	DEPLOYMENT_ENV=${DEPLOYMENT_ENV} \
+	IMAGE_TAG=${IMAGE_TAG} \
+		./scripts/jwt-issuer-up.sh
+
+jwt-issuer-down:
+	@DEPLOYMENT_ENV=${DEPLOYMENT_ENV} \
+		./scripts/jwt-issuer-down.sh
+
+jwt-issuer-trust:
+	@WORKSPACE=${KMS_WORKSPACE} \
+	KMS_URL=${KMS_URL} \
+	DEPLOYMENT_ENV=${DEPLOYMENT_ENV} \
+		./scripts/jwt-issuer-trust.sh
+
+# Manage KMS -------------------------------------------------------------------
+
+js-app-set:
+	@WORKSPACE=${KMS_WORKSPACE} \
+	KMS_URL=${KMS_URL} \
+		./scripts/js-app-set.sh
+
+constitution-set:
+	@WORKSPACE=${KMS_WORKSPACE} \
+	KMS_URL=${KMS_URL} \
+	CONSTITUTION_PATH=./governance/constitution/kms_actions.js \
+		./scripts/constitution-set.sh
+
+release-policy-set:
+	@WORKSPACE=${KMS_WORKSPACE} \
+	KMS_URL=${KMS_URL} \
+	RELEASE_POLICY_PROPOSAL=$(release-policy-proposal) \
+		./scripts/release-policy-set.sh
+
+test-system:
+	@pytest -s test/system-test/$(filter-out $@,$(MAKECMDGOALS))
+
 # Keep this at the bottom.
 clean: ## 🧹 Clean the working folders created during build/demo
 	@rm -rf ${PYTHON_VENV}

README.md

@@ -138,14 +138,14 @@ make setup
 ## Manual tests
 
 ```
-# Testing with hearthbeat: Use user certs
-curl ${KMS_URL}/app/hearthbeat --cacert ${KEYS_DIR}/service_cert.pem --cert ${KEYS_DIR}/user0_cert.pem --key ${KEYS_DIR}/user0_privk.pem -H "Content-Type: application/json" -w '\n' | jq
+# Testing with heartbeat: Use user certs
+curl ${KMS_URL}/app/heartbeat --cacert ${KEYS_DIR}/service_cert.pem --cert ${KEYS_DIR}/user0_cert.pem --key ${KEYS_DIR}/user0_privk.pem -H "Content-Type: application/json" -w '\n' | jq
 
-# Testing with hearthbeat: Use member certs
-curl ${KMS_URL}/app/hearthbeat --cacert ${KEYS_DIR}/service_cert.pem --cert ${KEYS_DIR}/member0_cert.pem --key ${KEYS_DIR}/member0_privk.pem -H "Content-Type: application/json" -w '\n' | jq
+# Testing with heartbeat: Use member certs
+curl ${KMS_URL}/app/heartbeat --cacert ${KEYS_DIR}/service_cert.pem --cert ${KEYS_DIR}/member0_cert.pem --key ${KEYS_DIR}/member0_privk.pem -H "Content-Type: application/json" -w '\n' | jq
 
-# Testing with hearthbeat: Use JWT
-curl ${KMS_URL}/app/hearthbeat --cacert ${KEYS_DIR}/service_cert.pem  -H "Content-Type: application/json" -H "Authorization:$AUTHORIZATION"  -w '\n' | jq
+# Testing with heartbeat: Use JWT
+curl ${KMS_URL}/app/heartbeat --cacert ${KEYS_DIR}/service_cert.pem  -H "Content-Type: application/json" -H "Authorization:$AUTHORIZATION"  -w '\n' | jq
 
 # Generate a new key item
 curl ${KMS_URL}/app/refresh -X POST --cacert ${KEYS_DIR}/service_cert.pem  -H "Content-Type: application/json" -i  -w '\n'
@@ -191,6 +191,12 @@ curl $KMS_URL/app/keyReleasePolicy --cacert ${KEYS_DIR}/service_cert.pem --cert
 curl $KMS_URL/receipt?transaction_id=2.20 --cacert ${KEYS_DIR}/service_cert.pem --cert ${KEYS_DIR}/user0_cert.pem --key ${KEYS_DIR}/user0_privk.pem -H "Content-Type: application/json" -i  -w '\n'
 ```
 
+## Run end to end system tests
+
+```
+pytest -s test/system-test
+```
+
 ## Access Tokens
 
 The manual curl test work with certificates. In this section we will use access tokens.

app.json

@@ -284,17 +284,33 @@
         }
       }
     },
-    "/hearthbeat": {
+    "/auth": {
       "get": {
         "js_module": "endpoints/kms.js",
-        "js_function": "hearthbeat",
-        "forwarding_required": "sometimes",
+        "js_function": "auth",
+        "forwarding_required": "never",
         "authn_policies": ["user_cert", "member_cert", "jwt"],
-        "mode": "readwrite",
+        "mode": "readonly",
+        "openapi": {
+          "responses": {
+            "200": {
+              "description": "Generate a heartbeat response"
+            }
+          }
+        }
+      }
+    },
+    "/heartbeat": {
+      "get": {
+        "js_module": "endpoints/kms.js",
+        "js_function": "heartbeat",
+        "forwarding_required": "sometimes",
+        "authn_policies": [],
+        "mode": "readonly",
         "openapi": {
           "responses": {
             "200": {
-              "description": "Generate a hearthbeat response"
+              "description": "Generate a heartbeat response"
             }
           }
         }