From 12227b4484d2176ef54d348804332f5a71405eb5 Mon Sep 17 00:00:00 2001 From: Alexei Dodon Date: Mon, 16 Oct 2023 11:51:00 +0300 Subject: [PATCH] fix: quick metrics setup in kind with prometheus (#1897) Signed-off-by: Alexei Dodon --- Makefile | 19 ++++- examples/metrics/Dockerfile | 40 ----------- examples/metrics/Dockerfile-minimal | 32 --------- examples/metrics/Dockerfile-zxp | 31 -------- examples/metrics/Makefile | 27 ------- examples/metrics/kind_setup.sh | 10 +-- .../kubernetes/prometheus/prom_rbac.yaml | 4 +- .../kubernetes/zot-extended/deployment.yaml | 71 ++++++++++++++++++- .../kubernetes/zot-minimal/deployment.yaml | 23 ++++++ 9 files changed, 118 insertions(+), 139 deletions(-) delete mode 100644 examples/metrics/Dockerfile delete mode 100644 examples/metrics/Dockerfile-minimal delete mode 100644 examples/metrics/Dockerfile-zxp delete mode 100644 examples/metrics/Makefile diff --git a/Makefile b/Makefile index e959ec3d..c14fe89a 100644 --- a/Makefile +++ b/Makefile @@ -259,7 +259,7 @@ swagger: .PHONY: update-licenses # note: for predictable output of below sort command we use locale LC_ALL=C update-licenses: LC_ALL=C -update-licenses: +update-licenses: check-linux @echo "Detecting and updating licenses ... please be patient!" go install github.com/google/go-licenses@latest $(shell echo "Module | License URL | License" > THIRD-PARTY-LICENSES.md; echo "---|---|---" >> THIRD-PARTY-LICENSES.md; for i in $$(go list -m all | awk '{print $$1}'); do l=$$(go-licenses csv $$i 2>/dev/null); if [ $$? -ne 0 ]; then continue; fi; echo $$l | tr \, \| | tr ' ' '\n'; done | sort -u >> THIRD-PARTY-LICENSES.md) @@ -362,6 +362,23 @@ run-container: ${CONTAINER_RUNTIME} run --rm --security-opt label=disable -v $$(pwd):/go/src/github.com/project-zot/zot \ zot-build:latest +.PHONY: binary-minimal-container +binary-minimal-container: + ${CONTAINER_RUNTIME} build ${BUILD_ARGS} -f build/Dockerfile-minimal -t zot-minimal:latest . + +.PHONY: run-minimal-container +run-minimal-container: + ${CONTAINER_RUNTIME} run --rm --security-opt label=disable -v $$(pwd):/go/src/github.com/project-zot/zot \ + zot-minimal:latest + +.PHONY: binary-exporter-container +binary-exporter-container: + ${CONTAINER_RUNTIME} build ${BUILD_ARGS} -f build/Dockerfile-zxp -t zxp:latest . + +.PHONY: run-exporter-container +run-exporter-container: + ${CONTAINER_RUNTIME} run --rm --security-opt label=disable zxp:latest + .PHONY: oci-image oci-image: $(STACKER) ${STACKER} --debug build \ diff --git a/examples/metrics/Dockerfile b/examples/metrics/Dockerfile deleted file mode 100644 index 1239a18c..00000000 --- a/examples/metrics/Dockerfile +++ /dev/null @@ -1,40 +0,0 @@ -# --- -# Stage 1: Install certs, build binary, create default config file -# --- -FROM ghcr.io/project-zot/golang:1.19 AS builder -RUN mkdir -p /go/src/github.com/project-zot/zot -WORKDIR /go/src/github.com/project-zot/zot -COPY . . -RUN make clean binary -RUN echo '{\n\ - "storage": {\n\ - "rootDirectory": "/var/lib/registry"\n\ - },\n\ - "http": {\n\ - "address": "0.0.0.0",\n\ - "port": "5000"\n\ - },\n\ - "log": {\n\ - "level": "debug"\n\ - },\n\ - "extensions": {\n\ - "metrics": {\n\ - "enable": true,\n\ - "prometheus": {\n\ - "path": "/metrics"\n\ - }\n\ - }\n\ - }\n\ -}\n' > config.json && cat config.json - -# --- -# Stage 2: Final image with nothing but certs, binary, and default config file -# --- -FROM gcr.io/distroless/base AS final -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -COPY --from=builder /go/src/github.com/project-zot/zot/bin/zot-linux-amd64 /usr/bin/zot -COPY --from=builder /go/src/github.com/project-zot/zot/config.json /etc/zot/config.json -ENTRYPOINT ["/usr/bin/zot"] -EXPOSE 5000 -VOLUME ["/var/lib/registry"] -CMD ["serve", "/etc/zot/config.json"] diff --git a/examples/metrics/Dockerfile-minimal b/examples/metrics/Dockerfile-minimal deleted file mode 100644 index d05d5606..00000000 --- a/examples/metrics/Dockerfile-minimal +++ /dev/null @@ -1,32 +0,0 @@ -# --- -# Stage 1: Install certs, build binary, create default config file -# --- -FROM ghcr.io/project-zot/golang:1.19 AS builder -RUN mkdir -p /go/src/github.com/project-zot/zot -WORKDIR /go/src/github.com/project-zot/zot -COPY . . -RUN make clean binary-minimal -RUN echo '{\n\ - "storage": {\n\ - "rootDirectory": "/var/lib/registry"\n\ - },\n\ - "http": {\n\ - "address": "0.0.0.0",\n\ - "port": "5050"\n\ - },\n\ - "log": {\n\ - "level": "debug"\n\ - }\n\ -}\n' > config.json && cat config.json - -# --- -# Stage 2: Final image with nothing but certs, binary, and default config file -# --- -FROM gcr.io/distroless/base AS final -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -COPY --from=builder /go/src/github.com/project-zot/zot/bin/zot-linux-amd64-minimal /usr/bin/zot -COPY --from=builder /go/src/github.com/project-zot/zot/config.json /etc/zot/config.json -ENTRYPOINT ["/usr/bin/zot"] -EXPOSE 5050 -VOLUME ["/var/lib/registry"] -CMD ["serve", "/etc/zot/config.json"] diff --git a/examples/metrics/Dockerfile-zxp b/examples/metrics/Dockerfile-zxp deleted file mode 100644 index ebe35685..00000000 --- a/examples/metrics/Dockerfile-zxp +++ /dev/null @@ -1,31 +0,0 @@ -# --- -# Stage 1: Build binary, create default config file -# --- -FROM ghcr.io/project-zot/golang:1.19 AS builder -RUN mkdir -p /go/src/github.com/project-zot/zot -WORKDIR /go/src/github.com/project-zot/zot -COPY . . -RUN make clean exporter-minimal -RUN echo '{\n\ - "Server": {\n\ - "protocol": "http",\n\ - "host": "127.0.0.1",\n\ - "port": "5050"\n\ - },\n\ - "Exporter": {\n\ - "port": "5051",\n\ - "log": {\n\ - "level": "debug"\n\ - }\n\ - }\n\ -}\n' > config.json && cat config.json - -# --- -# Stage 2: Final image with nothing but binary and default config file -# --- -FROM gcr.io/distroless/base AS final -COPY --from=builder /go/src/github.com/project-zot/zot/bin/zxp-linux-amd64 /usr/bin/zxp -COPY --from=builder /go/src/github.com/project-zot/zot/config.json /etc/zxp/config.json -ENTRYPOINT ["/usr/bin/zxp"] -EXPOSE 5051 -CMD ["config", "/etc/zxp/config.json"] diff --git a/examples/metrics/Makefile b/examples/metrics/Makefile deleted file mode 100644 index 2dbe13a9..00000000 --- a/examples/metrics/Makefile +++ /dev/null @@ -1,27 +0,0 @@ -CONTAINER_RUNTIME := docker - -.PHONY: binary-container -binary-container: - ${CONTAINER_RUNTIME} build -f build/Dockerfile -t zot-build:latest ../../. - -.PHONY: run-container -run-container: - ${CONTAINER_RUNTIME} run --rm --security-opt label=disable -v $$(pwd)/../..:/go/src/github.com/project-zot/zot \ - zot-build:latest - -.PHONY: binary-minimal-container -binary-minimal-container: - ${CONTAINER_RUNTIME} build -f build/Dockerfile-minimal -t zot-minimal:latest ../../. - -.PHONY: run-minimal-container -run-minimal-container: - ${CONTAINER_RUNTIME} run --rm --security-opt label=disable -v $$(pwd)/../..:/go/src/github.com/project-zot/zot \ - zot-minimal:latest - -.PHONY: binary-exporter-container -binary-exporter-container: - ${CONTAINER_RUNTIME} build -f build/Dockerfile-zxp -t zxp:latest ../../. - -.PHONY: run-exporter-container -run-exporter-container: - ${CONTAINER_RUNTIME} run --rm --security-opt label=disable zxp:latest diff --git a/examples/metrics/kind_setup.sh b/examples/metrics/kind_setup.sh index c52219a6..a6853ebd 100755 --- a/examples/metrics/kind_setup.sh +++ b/examples/metrics/kind_setup.sh @@ -6,8 +6,8 @@ set -e CLUSTER_NAME=zot # Script tested with below kubectl & kind versions -KUBECTL_VERSION=v1.17.5 -KIND_VERSION=v0.7.0 +KUBECTL_VERSION=v1.27.3 +KIND_VERSION=v0.20.0 function install_bin() { if [ "$EUID" -ne 0 ] @@ -41,9 +41,9 @@ kind load docker-image quay.io/prometheus-operator/prometheus-config-reloader:v0 kind load docker-image quay.io/prometheus/prometheus:v2.22.1 --name ${CLUSTER_NAME} ## Build zot & zxp images -make binary-container -make binary-minimal-container -make binary-exporter-container +make -C ../../ binary-container +make -C ../../ binary-minimal-container +make -C ../../ binary-exporter-container kind load docker-image zot-build:latest --name ${CLUSTER_NAME} kind load docker-image zot-minimal:latest --name ${CLUSTER_NAME} diff --git a/examples/metrics/kubernetes/prometheus/prom_rbac.yaml b/examples/metrics/kubernetes/prometheus/prom_rbac.yaml index c6bf0e8f..5b8fd88e 100644 --- a/examples/metrics/kubernetes/prometheus/prom_rbac.yaml +++ b/examples/metrics/kubernetes/prometheus/prom_rbac.yaml @@ -3,7 +3,7 @@ kind: ServiceAccount metadata: name: prometheus --- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: prometheus @@ -28,7 +28,7 @@ rules: - nonResourceURLs: ["/metrics"] verbs: ["get"] --- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: prometheus diff --git a/examples/metrics/kubernetes/zot-extended/deployment.yaml b/examples/metrics/kubernetes/zot-extended/deployment.yaml index 108fab8b..9e79d79a 100644 --- a/examples/metrics/kubernetes/zot-extended/deployment.yaml +++ b/examples/metrics/kubernetes/zot-extended/deployment.yaml @@ -18,7 +18,76 @@ spec: - name: zot-extended image: zot-build:latest imagePullPolicy: IfNotPresent + command: ["/usr/bin/zot"] + args: ["serve", "/zot-config/config.json"] ports: - name: zot-extended containerPort: 5000 - + protocol: TCP + volumeMounts: + - name: zot-config + mountPath: /zot-config + readOnly: false + volumes: + - name: zot-config + configMap: + name: zot-config + items: + - key: zot_config.json + path: config.json +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: zot-config +data: + zot_config.json: | + { + "distSpecVersion":"1.0.1", + "storage": { + "rootDirectory": "/var/lib/registry" + }, + "http": { + "address": "0.0.0.0", + "port": "5000" + }, + "log": { + "level": "debug" + }, + "extensions": { + "metrics": { + "enable": true, + "prometheus": { + "path": "/metrics" + } + } + } + } + zot_minimal.json: | + { + "distSpecVersion":"1.0.1", + "storage": { + "rootDirectory": "/var/lib/registry" + }, + "http": { + "address": "0.0.0.0", + "port": "5050" + }, + "log": { + "level": "debug" + } + } + zxp.json: | + { + "Server": { + "protocol": "http", + "host": "127.0.0.1", + "port": "5050" + }, + "Exporter": { + "port": "5051", + "log": { + "level": "debug" + } + } + } diff --git a/examples/metrics/kubernetes/zot-minimal/deployment.yaml b/examples/metrics/kubernetes/zot-minimal/deployment.yaml index c9aeccf8..99c54858 100644 --- a/examples/metrics/kubernetes/zot-minimal/deployment.yaml +++ b/examples/metrics/kubernetes/zot-minimal/deployment.yaml @@ -18,13 +18,36 @@ spec: - name: zot-minimal image: zot-minimal:latest imagePullPolicy: IfNotPresent + command: ["/usr/bin/zot"] + args: ["serve", "/zot-config/config.json"] ports: - name: zot-minimal containerPort: 5050 + protocol: TCP + volumeMounts: + - name: zot-config + mountPath: /zot-config + readOnly: false - name: zot-exporter image: zxp:latest imagePullPolicy: IfNotPresent + command: ["/zxp"] + args: ["config", "/zot-config/zxp-config.json"] ports: - name: zot-exporter containerPort: 5051 + protocol: TCP + volumeMounts: + - name: zot-config + mountPath: /zot-config + readOnly: false + volumes: + - name: zot-config + configMap: + name: zot-config + items: + - key: zot_minimal.json + path: config.json + - key: zxp.json + path: zxp-config.json