ci(nightly): add prometheus kind test (#1940)

Signed-off-by: Alexei Dodon <adodon@cisco.com>

.github/workflows/nightly.yaml

@@ -14,10 +14,9 @@ permissions: read-all
 jobs:
   dedupe:
     name: Dedupe/restore blobs
-    runs-on: ubuntu-latest-16-cores
+    runs-on: ubuntu-latest-8-cores
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/clean-runner
       - uses: actions/setup-go@v3
         with:
           go-version: 1.20.x
@@ -62,11 +61,10 @@ jobs:
 
   sync:
     name: Sync harness
-    runs-on: ubuntu-latest-16-cores
+    runs-on: ubuntu-latest-8-cores
     steps:
       - name: Check out source code
         uses: actions/checkout@v4
-      - uses: ./.github/actions/clean-runner
       - uses: actions/setup-go@v3
         with:
           go-version: 1.20.x
@@ -84,7 +82,6 @@ jobs:
     runs-on: ubuntu-latest-16-cores
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/clean-runner
       - uses: actions/setup-go@v4
         with:
           cache: false
@@ -92,6 +89,7 @@ jobs:
       - uses: ./.github/actions/setup-localstack
 
       - name: Run zb
+        timeout-minutes: 240
         id: bench
         run: |
             make binary
@@ -121,7 +119,6 @@ jobs:
     runs-on: ubuntu-latest-16-cores
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/clean-runner
       - uses: actions/setup-go@v4
         with:
           cache: false
@@ -129,6 +126,7 @@ jobs:
       - uses: ./.github/actions/setup-localstack
 
       - name: Run zb
+        timeout-minutes: 240
         id: bench
         run: |
             make binary
@@ -155,12 +153,41 @@ jobs:
 
   docker-image:
     name: Build docker image (for users still using Docker environments)
-    runs-on: ubuntu-latest-16-cores
+    runs-on: ubuntu-latest-4-cores
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/clean-runner
       - name: Check out source code
         uses: actions/checkout@v4
       - name: Build image
         run: |
           make docker-image
+
+  kind-setup:
+    name: Prometheus setup
+    runs-on: ubuntu-latest-8-cores
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-go@v3
+        with:
+          go-version: 1.20.x
+      - name: Install dependencies
+        run: |
+          cd $GITHUB_WORKSPACE
+          go mod download
+          sudo apt-get update
+          sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config rpm uidmap
+          # install skopeo
+          git clone -b v1.12.0 https://github.com/containers/skopeo.git
+          cd skopeo
+          make bin/skopeo
+          sudo cp bin/skopeo /usr/bin
+          skopeo -v
+      - name: Log in to GitHub Docker Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+      - name: Run tests
+        run: |
+            ./examples/kind/kind-ci.sh

examples/kind/kind-ci.sh

@@ -0,0 +1,108 @@
+#!/bin/sh
+set -o errexit
+
+# Reference: https://kind.sigs.k8s.io/docs/user/local-registry/
+
+# set no_proxy if applicable
+if [ ! -z "${no_proxy}" ]; then
+  echo "Updating no_proxy env var";
+  export no_proxy=${no_proxy},kind-registry;
+  export NO_PROXY=${no_proxy};
+fi
+
+# create registry container unless it already exists
+reg_name='kind-registry'
+reg_port='5001'
+if [ "$(docker inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
+  docker run \
+    -d --restart=always -p "127.0.0.1:${reg_port}:5000" --name "${reg_name}" \
+    ghcr.io/project-zot/zot-minimal-linux-amd64:latest
+fi
+
+CLUSTER_NAME=kind
+## Delete the cluster if it already exist
+kind get clusters | grep ${CLUSTER_NAME} &&  kind delete cluster --name ${CLUSTER_NAME}
+
+# create a cluster with the local registry enabled in containerd
+cat <<EOF | kind create cluster --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:${reg_port}"]
+    endpoint = ["http://${reg_name}:5000"]
+EOF
+
+# connect the registry to the cluster network if not already connected
+if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
+  docker network connect "kind" "${reg_name}"
+fi
+
+# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
+#
+# document the local registry
+cat <<EOF | kubectl apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:${reg_port}"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
+EOF
+
+## Deploy prometheus operator
+kubectl create -f examples/metrics/kubernetes/prometheus/bundle.yaml
+
+## Deploy the Kubernetes objects for RBAC, prometheus CRD and deploy the service
+kubectl apply -f examples/metrics/kubernetes/prometheus/prom_rbac.yaml
+kubectl apply -f examples/metrics/kubernetes/prometheus/prometheus.yaml
+kubectl apply -f examples/metrics/kubernetes/prometheus/prom_service.yaml
+
+make oci-image
+# copy the image
+COMMIT_HASH=$(git describe --always --tags --long)
+echo "deploy zot-build:${COMMIT_HASH} image to local registry"
+skopeo copy --format=oci --dest-tls-verify=false oci:oci docker://localhost:5001/zot-build:${COMMIT_HASH}
+
+# deploy the image
+kubectl apply -f examples/metrics/kubernetes/zot-extended/deployment.yaml
+kubectl patch deployment/zot-extended --patch-file examples/metrics/kubernetes/zot-extended/patch-deployment.yaml
+kubectl set image deployment/zot-extended zot-extended=localhost:5001/zot-build:${COMMIT_HASH}
+kubectl apply -f examples/metrics/kubernetes/zot-extended/service.yaml
+kubectl apply -f examples/metrics/kubernetes/zot-extended/servicemonitor.yaml
+
+# check for availability
+echo "Waiting for deployment/zot-extended to be ready ..."
+kubectl wait deployment -n default zot-extended --for condition=Available=True --timeout=90s
+kubectl wait deployment -n default prometheus-operator --for condition=Available=True --timeout=90s
+
+kubectl port-forward svc/prometheus 9090 --address='0.0.0.0' &
+echo "Kind cluster status before sleep:"
+kubectl get pods -A
+# Put enough amount of time for prometheus scraping take place
+sleep 90
+echo "Kind cluster status:"
+kubectl get pods -A
+echo "zot-extended logs:"
+kubectl logs -l app=zot-extended --tail=-1
+
+containername=`curl -s http://localhost:9090/api/v1/query?query=up | jq '.data.result[].metric.container'`
+echo "containername=${containername}"
+if [ "${containername}" != '"zot-extended"' ]; then
+    exit 1
+fi
+
+containerup=`curl -s http://localhost:9090/api/v1/query?query=up | jq '.data.result[].value[1]'`
+echo "containerup=${containerup}"
+if [ "${containerup}" != '"1"' ]; then
+    exit 1
+fi
+
+zotinfo=`curl -s http://localhost:9090/api/v1/query?query=zot_info | jq '.data.result[].value[1]'`
+echo "zotinfo=${zotinfo}"
+if [ "${zotinfo}" != '"0"' ]; then
+    exit 1
+fi

examples/metrics/kubernetes/zot-extended/deployment.yaml

@@ -35,6 +35,8 @@ spec:
             items:
             - key: zot_config.json
               path: config.json
+            - key: htpasswd
+              path: htpasswd
 ---
 apiVersion: v1
 kind: ConfigMap
@@ -49,7 +51,25 @@ data:
       },
       "http": {
           "address": "0.0.0.0",
-          "port": "5000"
+          "port": "5000",
+          "auth": {
+              "htpasswd": {
+                "path": "/zot-config/htpasswd"
+              }
+          },
+          "accessControl": {
+              "metrics":{
+                  "users": ["metrics"]
+              },
+              "repositories": {
+                  "**": {
+                      "anonymousPolicy": [
+                          "read"
+                      ],
+                      "defaultPolicy": ["read","create"]
+                  }
+              }
+          }
       },
       "log": {
           "level": "debug"
@@ -91,3 +111,7 @@ data:
         }
       }
     }
+  # Example htpasswd with 'test:test' & 'metrics:metrics' user:pass pairs
+  htpasswd: |-
+    test:$2y$05$.jSWenVyzEK3em/Gfr0AG.WRSAIdi4nxqW9h27xK7WCw24wmQH/1m
+    metrics:$2y$05$4yBka/ZTKgXhvCMb48BnyOZqj/DrKT1sGPZLAg5RbobQ0CQCJHmTO

examples/metrics/kubernetes/zot-extended/patch-deployment.yaml

@@ -0,0 +1,7 @@
+spec:
+  template:
+    spec:
+      containers:
+      - name: zot-extended
+        command: ["/usr/local/bin/zot-linux-amd64"]
+

examples/metrics/kubernetes/zot-extended/servicemonitor.yaml

@@ -9,7 +9,23 @@ spec:
   - interval: 10s
     port: zot-extended
     scrapeTimeout: 5s
+    basicAuth:
+      password:
+        name: basic-auth
+        key: password
+      username:
+        name: basic-auth
+        key: user
   selector:
     matchLabels:
       app: zot-extended
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: basic-auth
+data:
+  password: bWV0cmljcw== # metrics
+  user: bWV0cmljcw==     # metrics
+type: Opaque