test: additional blackbox tests for client push/pull (#1371)

Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>

.github/workflows/ecosystem-tools.yaml

@@ -34,6 +34,18 @@ jobs:
           make bin/skopeo
           sudo cp bin/skopeo /usr/bin
           skopeo -v
+          # install cri-o (for crictl)
+          OS=xUbuntu_20.04
+          CRIO_VERSION=1.26
+          echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /"|sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
+          echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$CRIO_VERSION/$OS/ /"|sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$CRIO_VERSION.list
+          curl -L https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable:cri-o:$CRIO_VERSION/$OS/Release.key | sudo apt-key add -
+          curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | sudo apt-key add -
+          sudo apt update
+          sudo apt install -y cri-o cri-o-runc
+          sudo systemctl enable crio.service
+          sudo systemctl start crio.service
+          sudo chmod 0777 /var/run/crio/crio.sock
       - name: Run referrers tests
         run: |
             make test-bats-referrers

Makefile

@@ -19,6 +19,8 @@ ORAS := $(TOOLSDIR)/bin/oras
 ORAS_VERSION := 1.0.0-rc.1
 REGCLIENT := $(TOOLSDIR)/bin/regctl
 REGCLIENT_VERSION := v0.4.5
+CRICTL := $(TOOLSDIR)/bin/crictl
+CRICTL_VERSION := v1.26.1
 ACTION_VALIDATOR := $(TOOLSDIR)/bin/action-validator
 ACTION_VALIDATOR_VERSION := v0.2.1
 ZUI_VERSION := commit-2f94cc3
@@ -144,6 +146,14 @@ $(REGCLIENT):
 	cp regctl $(TOOLSDIR)/bin/regctl
 	chmod +x $(TOOLSDIR)/bin/regctl
 
+$(CRICTL):
+	mkdir -p $(TOOLSDIR)/bin
+	curl -Lo crictl.tar.gz https://github.com/kubernetes-sigs/cri-tools/releases/download/$(CRICTL_VERSION)/crictl-$(CRICTL_VERSION)-linux-amd64.tar.gz
+	tar xvzf crictl.tar.gz
+	cp crictl $(TOOLSDIR)/bin/crictl
+	chmod +x $(TOOLSDIR)/bin/crictl
+
+
 $(ACTION_VALIDATOR):
 	mkdir -p $(TOOLSDIR)/bin
 	curl -Lo action-validator https://github.com/mpalmer/action-validator/releases/download/$(ACTION_VALIDATOR_VERSION)/action-validator_linux_amd64
@@ -300,7 +310,7 @@ $(BATS):
 	rm -rf bats-core
 
 .PHONY: test-push-pull
-test-push-pull: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
+test-push-pull: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
 	$(BATS) --trace --print-output-on-failure test/blackbox/pushpull.bats
 
 .PHONY: test-push-pull-verbose
@@ -308,19 +318,19 @@ test-push-pull-verbose: binary check-skopeo $(BATS)
 	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull.bats
 
 .PHONY: test-push-pull-running-dedupe
-test-push-pull-running-dedupe: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
+test-push-pull-running-dedupe: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
 	$(BATS) --trace --print-output-on-failure test/blackbox/pushpull_running_dedupe.bats
 
 .PHONY: test-push-pull-running-dedupe-verbose
-test-push-pull-running-dedupe-verbose: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
+test-push-pull-running-dedupe-verbose: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
 	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull_running_dedupe.bats
 
 .PHONY: test-restore-s3-blobs
-test-restore-s3-blobs: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
+test-restore-s3-blobs: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
 	$(BATS) --trace --print-output-on-failure test/blackbox/restore_s3_blobs.bats
 
 .PHONY: test-restore-s3-blobs-verbose
-test-restore-s3-blobs-verbose: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
+test-restore-s3-blobs-verbose: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
 	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/restore_s3_blobs.bats
 
 .PHONY: test-bats-referrers

test/blackbox/pushpull.bats

@@ -203,6 +203,16 @@ EOF
     [ "$status" -eq 0 ]
 }
 
+@test "pull manifest with docker client" {
+    run docker pull localhost:8080/test-regclient
+    [ "$status" -eq 0 ]
+}
+
+@test "pull manifest with crictl" {
+    run crictl pull localhost:8080/test-regclient
+    [ "$status" -eq 0 ]
+}
+
 @test "push OCI artifact with regclient" {
     run regctl artifact put localhost:8080/artifact:demo <<EOF
 this is an artifact