Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-16 21:56:37 -05:00
Code Issues Activity

feat(repodb): Multiarch Image support (#1147)

Browse source 
* feat(repodb): index logic + tests

Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>

* feat(cli): printing indexes support using the rest api

Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>

---------

Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
This commit is contained in:
LaurentiuNiculae 2023-02-27 21:23:18 +02:00 committed by GitHub
parent a561d0bad5
commit d62c09e2cc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
54 changed files with 8656 additions and 2988 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
errors/errors.go
View file

@ -63,6 +63,7 @@ var (
ErrManifestConflict = errors.New("manifest: multiple manifests found")
ErrManifestMetaNotFound = errors.New("repodb: image metadata not found for given manifest digest")
ErrManifestDataNotFound = errors.New("repodb: image data not found for given manifest digest")
ErrIndexDataNotFount = errors.New("repodb: index data not found for given digest")
ErrRepoMetaNotFound = errors.New("repodb: repo metadata not found for given repo name")
ErrTagMetaNotFound = errors.New("repodb: tag metadata not found for given repo and tag names")
ErrTypeAssertionFailed = errors.New("storage: failed DatabaseDriver type assertion")
@ -77,4 +78,5 @@ var (
ErrOffsetIsNegative = errors.New("pageturner: offset has negative value")
ErrSortCriteriaNotSupported = errors.New("pageturner: the sort criteria is not supported")
ErrTimeout = errors.New("operation timeout")
ErrNotImplemented = errors.New("not implemented")
)

4
pkg/api/controller.go
View file

@ -551,6 +551,9 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) d
manifestDataTablename, ok := toStringIfOk(cacheDriverConfig, "manifestdatatablename", log)
allParametersOk = allParametersOk && ok
indexDataTablename, ok := toStringIfOk(cacheDriverConfig, "indexdatatablename", log)
allParametersOk = allParametersOk && ok
versionTablename, ok := toStringIfOk(cacheDriverConfig, "versiontablename", log)
allParametersOk = allParametersOk && ok
@ -563,6 +566,7 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) d
Region: region,
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
VersionTablename: versionTablename,
}
}

137
pkg/api/controller_test.go
View file

@ -3860,10 +3860,10 @@ func TestImageSignatures(t *testing.T) {
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "1.0",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "1.0",
}, baseURL, repoName)
So(err, ShouldBeNil)
@ -4100,10 +4100,10 @@ func TestArtifactReferences(t *testing.T) {
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "1.0",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "1.0",
}, baseURL, repoName)
So(err, ShouldBeNil)
@ -4967,10 +4967,10 @@ func TestStorageCommit(t *testing.T) {
repoName := "repo7"
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "test:1.0",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "test:1.0",
}, baseURL, repoName)
So(err, ShouldBeNil)
@ -5002,10 +5002,10 @@ func TestStorageCommit(t *testing.T) {
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "test:1.0.1",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "test:1.0.1",
}, baseURL, repoName)
So(err, ShouldBeNil)
@ -5014,10 +5014,10 @@ func TestStorageCommit(t *testing.T) {
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "test:2.0",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "test:2.0",
}, baseURL, repoName)
So(err, ShouldBeNil)
@ -5124,10 +5124,10 @@ func TestManifestImageIndex(t *testing.T) {
repoName := "index"
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "test:1.0",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "test:1.0",
}, baseURL, repoName)
So(err, ShouldBeNil)
@ -5552,10 +5552,10 @@ func TestManifestCollision(t *testing.T) {
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "test:1.0",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "test:1.0",
}, baseURL, "index")
So(err, ShouldBeNil)
@ -5579,10 +5579,10 @@ func TestManifestCollision(t *testing.T) {
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "test:2.0",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "test:2.0",
}, baseURL, "index")
So(err, ShouldBeNil)
@ -6215,10 +6215,10 @@ func TestGCSignaturesAndUntaggedManifests(t *testing.T) {
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: tag,
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: tag,
}, baseURL, repoName)
So(err, ShouldNotBeNil)
@ -6232,10 +6232,10 @@ func TestGCSignaturesAndUntaggedManifests(t *testing.T) {
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: tag,
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: tag,
}, baseURL, repoName)
So(err, ShouldNotBeNil)
@ -6253,10 +6253,10 @@ func TestGCSignaturesAndUntaggedManifests(t *testing.T) {
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: untaggedManifestDigest.String(),
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: untaggedManifestDigest.String(),
}, baseURL, repoName)
So(err, ShouldBeNil)
@ -6266,10 +6266,10 @@ func TestGCSignaturesAndUntaggedManifests(t *testing.T) {
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: tag,
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: tag,
}, baseURL, repoName)
So(err, ShouldBeNil)
@ -6342,10 +6342,10 @@ func TestGCSignaturesAndUntaggedManifests(t *testing.T) {
err = test.UploadImage(
test.Image{
Manifest: manifest,
Config: config,
Layers: layers,
Tag: manifestDigest.String(),
Manifest: manifest,
Config: config,
Layers: layers,
Reference: manifestDigest.String(),
},
baseURL,
repoName)
@ -6527,10 +6527,10 @@ func TestSearchRoutes(t *testing.T) {
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "latest",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "latest",
}, baseURL, repoName)
So(err, ShouldBeNil)
@ -6541,10 +6541,10 @@ func TestSearchRoutes(t *testing.T) {
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "latest",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "latest",
}, baseURL, inaccessibleRepo)
So(err, ShouldBeNil)
@ -6615,10 +6615,10 @@ func TestSearchRoutes(t *testing.T) {
err = test.UploadImageWithBasicAuth(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "latest",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "latest",
}, baseURL, repoName,
user1, password1)
@ -6630,10 +6630,10 @@ func TestSearchRoutes(t *testing.T) {
err = test.UploadImageWithBasicAuth(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "latest",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "latest",
}, baseURL, inaccessibleRepo,
user1, password1)
@ -6657,7 +6657,6 @@ func TestSearchRoutes(t *testing.T) {
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
So(string(resp.Body()), ShouldContainSubstring, repoName)
So(string(resp.Body()), ShouldNotContainSubstring, inaccessibleRepo)

369
pkg/cli/client.go
View file

@ -52,6 +52,19 @@ func makeGETRequest(ctx context.Context, url, username, password string,
return doHTTPRequest(req, verifyTLS, debug, resultsPtr, configWriter)
}
func makeHEADRequest(ctx context.Context, url, username, password string, verifyTLS bool,
debug bool,
) (http.Header, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
if err != nil {
return nil, err
}
req.SetBasicAuth(username, password)
return doHTTPRequest(req, verifyTLS, debug, nil, io.Discard)
}
func makeGraphQLRequest(ctx context.Context, url, query, username,
password string, verifyTLS bool, debug bool, resultsPtr interface{}, configWriter io.Writer,
) error {
@ -126,6 +139,10 @@ func doHTTPRequest(req *http.Request, verifyTLS bool, debug bool,
return nil, errors.New(string(bodyBytes)) //nolint: goerr113
}
if resultsPtr == nil {
return resp.Header, nil
}
if err := json.NewDecoder(resp.Body).Decode(resultsPtr); err != nil {
return nil, err
}
@ -140,26 +157,25 @@ func isURL(str string) bool {
} // from https://stackoverflow.com/a/55551215
type requestsPool struct {
jobs chan *manifestJob
jobs chan *httpJob
done chan struct{}
wtgrp *sync.WaitGroup
outputCh chan stringResult
}
type manifestJob struct {
url string
username string
password string
imageName string
tagName string
config searchConfig
manifestResp manifestResponse
type httpJob struct {
url string
username string
password string
imageName string
tagName string
config searchConfig
}
const rateLimiterBuffer = 5000
func newSmoothRateLimiter(wtgrp *sync.WaitGroup, opch chan stringResult) *requestsPool {
ch := make(chan *manifestJob, rateLimiterBuffer)
ch := make(chan *httpJob, rateLimiterBuffer)
return &requestsPool{
jobs: ch,
@ -188,11 +204,12 @@ func (p *requestsPool) startRateLimiter(ctx context.Context) {
}
}
func (p *requestsPool) doJob(ctx context.Context, job *manifestJob) {
func (p *requestsPool) doJob(ctx context.Context, job *httpJob) {
defer p.wtgrp.Done()
header, err := makeGETRequest(ctx, job.url, job.username, job.password,
*job.config.verifyTLS, *job.config.debug, &job.manifestResp, job.config.resultWriter)
// Check manifest media type
header, err := makeHEADRequest(ctx, job.url, job.username, job.password, *job.config.verifyTLS,
*job.config.debug)
if err != nil {
if isContextDone(ctx) {
return
@ -200,88 +217,298 @@ func (p *requestsPool) doJob(ctx context.Context, job *manifestJob) {
p.outputCh <- stringResult{"", err}
}
digestStr := header.Get("docker-content-digest")
configDigest := job.manifestResp.Config.Digest
switch header.Get("Content-Type") {
case ispec.MediaTypeImageManifest:
image, err := fetchImageManifestStruct(ctx, job)
if err != nil {
if isContextDone(ctx) {
return
}
p.outputCh <- stringResult{"", err}
var size uint64
return
}
platformStr := getPlatformStr(image.Manifests[0].Platform)
str, err := image.string(*job.config.outputFormat, len(job.imageName), len(job.tagName), len(platformStr))
if err != nil {
if isContextDone(ctx) {
return
}
p.outputCh <- stringResult{"", err}
return
}
if isContextDone(ctx) {
return
}
p.outputCh <- stringResult{str, nil}
case ispec.MediaTypeImageIndex:
image, err := fetchImageIndexStruct(ctx, job)
if err != nil {
if isContextDone(ctx) {
return
}
p.outputCh <- stringResult{"", err}
return
}
platformStr := getPlatformStr(image.Manifests[0].Platform)
str, err := image.string(*job.config.outputFormat, len(job.imageName), len(job.tagName), len(platformStr))
if err != nil {
if isContextDone(ctx) {
return
}
p.outputCh <- stringResult{"", err}
return
}
if isContextDone(ctx) {
return
}
p.outputCh <- stringResult{str, nil}
default:
return
}
}
func fetchImageIndexStruct(ctx context.Context, job *httpJob) (*imageStruct, error) {
var indexContent ispec.Index
header, err := makeGETRequest(ctx, job.url, job.username, job.password,
*job.config.verifyTLS, *job.config.debug, &indexContent, job.config.resultWriter)
if err != nil {
if isContextDone(ctx) {
return nil, context.Canceled
}
return nil, err
}
indexDigest := header.Get("docker-content-digest")
indexSize, err := strconv.ParseInt(header.Get("Content-Length"), 10, 64)
if err != nil {
return nil, err
}
imageSize := indexSize
manifestList := make([]manifestStruct, 0, len(indexContent.Manifests))
for _, manifestDescriptor := range indexContent.Manifests {
manifest, err := fetchManifestStruct(ctx, job.imageName, manifestDescriptor.Digest.String(),
job.config, job.username, job.password)
if err != nil {
return nil, err
}
imageSize += int64(atoiWithDefault(manifest.Size, 0))
if manifestDescriptor.Platform != nil {
manifest.Platform = platform{
Os: manifestDescriptor.Platform.OS,
Arch: manifestDescriptor.Platform.Architecture,
Variant: manifestDescriptor.Platform.Variant,
}
}
manifestList = append(manifestList, manifest)
}
isIndexSigned := isCosignSigned(ctx, job.imageName, indexDigest, job.config, job.username, job.password) ||
isNotationSigned(ctx, job.imageName, indexDigest, job.config, job.username, job.password)
return &imageStruct{
verbose: *job.config.verbose,
RepoName: job.imageName,
Tag: job.tagName,
Size: strconv.FormatInt(imageSize, 10),
IsSigned: isIndexSigned,
Manifests: manifestList,
}, nil
}
func atoiWithDefault(size string, defaultVal int) int {
val, err := strconv.Atoi(size)
if err != nil {
return defaultVal
}
return val
}
func fetchImageManifestStruct(ctx context.Context, job *httpJob) (*imageStruct, error) {
manifest, err := fetchManifestStruct(ctx, job.imageName, job.tagName, job.config, job.username, job.password)
if err != nil {
return nil, err
}
return &imageStruct{
verbose: *job.config.verbose,
RepoName: job.imageName,
Tag: job.tagName,
Size: manifest.Size,
IsSigned: manifest.IsSigned,
Manifests: []manifestStruct{
manifest,
},
}, nil
}
func fetchManifestStruct(ctx context.Context, repo, manifestReference string, searchConf searchConfig,
username, password string,
) (manifestStruct, error) {
manifestResp := ispec.Manifest{}
URL := fmt.Sprintf("%s/v2/%s/manifests/%s",
*searchConf.servURL, repo, manifestReference)
header, err := makeGETRequest(ctx, URL, username, password,
*searchConf.verifyTLS, *searchConf.debug, &manifestResp, searchConf.resultWriter)
if err != nil {
if isContextDone(ctx) {
return manifestStruct{}, context.Canceled
}
return manifestStruct{}, err
}
manifestDigest := header.Get("docker-content-digest")
configDigest := manifestResp.Config.Digest.String()
configContent, err := fetchConfig(ctx, repo, configDigest, searchConf, username, password)
if err != nil {
if isContextDone(ctx) {
return manifestStruct{}, context.Canceled
}
return manifestStruct{}, err
}
opSys := ""
arch := ""
variant := ""
if manifestResp.Config.Platform != nil {
opSys = manifestResp.Config.Platform.OS
arch = manifestResp.Config.Platform.Architecture
variant = manifestResp.Config.Platform.Variant
}
if opSys == "" {
opSys = configContent.OS
}
if arch == "" {
arch = configContent.Architecture
}
if variant == "" {
variant = configContent.Variant
}
manifestSize, err := strconv.ParseInt(header.Get("Content-Length"), 10, 64)
if err != nil {
return manifestStruct{}, err
}
var imageSize int64
imageSize += manifestResp.Config.Size
imageSize += manifestSize
layers := []layer{}
for _, entry := range job.manifestResp.Layers {
size += entry.Size
for _, entry := range manifestResp.Layers {
imageSize += entry.Size
layers = append(
layers,
layer{
Size: entry.Size,
Digest: entry.Digest,
Digest: entry.Digest.String(),
},
)
}
size += uint64(job.manifestResp.Config.Size)
isSigned := isCosignSigned(ctx, repo, manifestDigest, searchConf, username, password) ||
isNotationSigned(ctx, repo, manifestDigest, searchConf, username, password)
manifestSize, err := strconv.Atoi(header.Get("Content-Length"))
if err != nil {
p.outputCh <- stringResult{"", err}
}
return manifestStruct{
ConfigDigest: configDigest,
Digest: manifestDigest,
Layers: layers,
Platform: platform{Os: opSys, Arch: arch, Variant: variant},
Size: strconv.FormatInt(imageSize, 10),
IsSigned: isSigned,
}, nil
}
isSigned := false
cosignTag := strings.Replace(digestStr, ":", "-", 1) + "." + remote.SignatureTagSuffix
func fetchConfig(ctx context.Context, repo, configDigest string, searchConf searchConfig,
username, password string,
) (ispec.Image, error) {
configContent := ispec.Image{}
_, err = makeGETRequest(ctx, *job.config.servURL+"/v2/"+job.imageName+
"/manifests/"+cosignTag, job.username, job.password,
*job.config.verifyTLS, *job.config.debug, &job.manifestResp, job.config.resultWriter)
if err == nil {
isSigned = true
}
URL := fmt.Sprintf("%s/v2/%s/blobs/%s",
*searchConf.servURL, repo, configDigest)
var referrers ispec.Index
if !isSigned {
_, err = makeGETRequest(ctx, fmt.Sprintf("%s/v2/%s/referrers/%s?artifactType=%s",
*job.config.servURL, job.imageName, digestStr, notreg.ArtifactTypeNotation), job.username, job.password,
*job.config.verifyTLS, *job.config.debug, &referrers, job.config.resultWriter)
if err == nil {
for _, reference := range referrers.Manifests {
if reference.ArtifactType == notreg.ArtifactTypeNotation {
isSigned = true
break
}
}
}
}
size += uint64(manifestSize)
image := &imageStruct{}
image.verbose = *job.config.verbose
image.RepoName = job.imageName
image.Tag = job.tagName
image.Digest = digestStr
image.Size = strconv.Itoa(int(size))
image.ConfigDigest = configDigest
image.Layers = layers
image.IsSigned = isSigned
str, err := image.string(*job.config.outputFormat, len(job.imageName), len(job.tagName))
_, err := makeGETRequest(ctx, URL, username, password,
*searchConf.verifyTLS, *searchConf.debug, &configContent, searchConf.resultWriter)
if err != nil {
if isContextDone(ctx) {
return
return ispec.Image{}, context.Canceled
}
p.outputCh <- stringResult{"", err}
return
return ispec.Image{}, err
}
if isContextDone(ctx) {
return
}
p.outputCh <- stringResult{str, nil}
return configContent, nil
}
func (p *requestsPool) submitJob(job *manifestJob) {
func isNotationSigned(ctx context.Context, repo, digestStr string, searchConf searchConfig,
username, password string,
) bool {
var referrers ispec.Index
URL := fmt.Sprintf("%s/v2/%s/referrers/%s?artifactType=%s",
*searchConf.servURL, repo, digestStr, notreg.ArtifactTypeNotation)
_, err := makeGETRequest(ctx, URL, username, password,
*searchConf.verifyTLS, *searchConf.debug, &referrers, searchConf.resultWriter)
if err != nil {
return false
}
for _, reference := range referrers.Manifests {
if reference.ArtifactType == notreg.ArtifactTypeNotation {
return true
}
}
return false
}
func isCosignSigned(ctx context.Context, repo, digestStr string, searchConf searchConfig,
username, password string,
) bool {
var result interface{}
cosignTag := strings.Replace(digestStr, ":", "-", 1) + "." + remote.SignatureTagSuffix
URL := fmt.Sprintf("%s/v2/%s/manifests/%s", *searchConf.servURL, repo, cosignTag)
_, err := makeGETRequest(ctx, URL, username, password, *searchConf.verifyTLS,
*searchConf.debug, &result, searchConf.resultWriter)
return err == nil
}
func (p *requestsPool) submitJob(job *httpJob) {
p.jobs <- job
}

652
pkg/cli/client_utils_test.go Normal file
View file

@ -0,0 +1,652 @@
//go:build search
// +build search
package cli //nolint:testpackage
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"sync"
"testing"
"github.com/gorilla/mux"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
. "github.com/smartystreets/goconvey/convey"
"zotregistry.io/zot/pkg/test"
)
type RouteHandler struct {
Route string
// HandlerFunc is the HTTP handler function that receives a writer for output and an HTTP request as input.
HandlerFunc http.HandlerFunc
// AllowedMethods specifies the HTTP methods allowed for the current route.
AllowedMethods []string
}
// Routes is a map that associates HTTP paths to their corresponding HTTP handlers.
type HTTPRoutes []RouteHandler
func StartTestHTTPServer(routes HTTPRoutes, port string) *http.Server {
baseURL := test.GetBaseURL(port)
mux := mux.NewRouter()
mux.HandleFunc("/test", func(w http.ResponseWriter, r *http.Request) {
_, err := w.Write([]byte("{}"))
if err != nil {
return
}
}).Methods(http.MethodGet)
for _, routeHandler := range routes {
mux.HandleFunc(routeHandler.Route, routeHandler.HandlerFunc).Methods(routeHandler.AllowedMethods...)
}
server := &http.Server{ //nolint:gosec
Addr: fmt.Sprintf(":%s", port),
Handler: mux,
}
go func() {
if err := server.ListenAndServe(); !errors.Is(err, http.ErrServerClosed) {
return
}
}()
test.WaitTillServerReady(baseURL + "/test")
return server
}
func getDefaultSearchConf(baseURL string) searchConfig {
verifyTLS := false
debug := false
verbose := true
outputFormat := "text"
return searchConfig{
servURL: &baseURL,
resultWriter: io.Discard,
verifyTLS: &verifyTLS,
debug: &debug,
verbose: &verbose,
outputFormat: &outputFormat,
}
}
func TestDoHTTPRequest(t *testing.T) {
Convey("doHTTPRequest nil result pointer", t, func() {
port := test.GetFreePort()
server := StartTestHTTPServer(nil, port)
defer server.Close()
url := fmt.Sprintf("http://127.0.0.1:%s/asd", port)
req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, url, nil)
So(err, ShouldBeNil)
So(func() { _, _ = doHTTPRequest(req, false, false, nil, io.Discard) }, ShouldNotPanic)
})
Convey("doHTTPRequest bad return json", t, func() {
port := test.GetFreePort()
server := StartTestHTTPServer(HTTPRoutes{
{
Route: "/test",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
_, err := w.Write([]byte("bad json"))
if err != nil {
return
}
},
AllowedMethods: []string{http.MethodGet},
},
}, port)
defer server.Close()
url := fmt.Sprintf("http://127.0.0.1:%s/test", port)
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil)
So(err, ShouldBeNil)
So(func() { _, _ = doHTTPRequest(req, false, false, &ispec.Manifest{}, io.Discard) }, ShouldNotPanic)
})
Convey("makeGraphQLRequest bad request context", t, func() {
err := makeGraphQLRequest(nil, "", "", "", "", false, false, nil, io.Discard) //nolint:staticcheck
So(err, ShouldNotBeNil)
})
Convey("makeHEADRequest bad request context", t, func() {
_, err := makeHEADRequest(nil, "", "", "", false, false) //nolint:staticcheck
So(err, ShouldNotBeNil)
})
Convey("makeGETRequest bad request context", t, func() {
_, err := makeGETRequest(nil, "", "", "", false, false, nil, io.Discard) //nolint:staticcheck
So(err, ShouldNotBeNil)
})
Convey("fetchImageManifestStruct errors", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
searchConf := getDefaultSearchConf(baseURL)
// 404 erorr will appear
server := StartTestHTTPServer(HTTPRoutes{}, port)
defer server.Close()
URL := baseURL + "/v2/repo/manifests/tag"
_, err := fetchImageManifestStruct(context.Background(), &httpJob{
url: URL,
username: "",
password: "",
imageName: "repo",
tagName: "tag",
config: searchConf,
})
So(err, ShouldNotBeNil)
})
Convey("fetchManifestStruct errors", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
searchConf := getDefaultSearchConf(baseURL)
Convey("makeGETRequest manifest error, context is done", func() {
server := StartTestHTTPServer(HTTPRoutes{}, port)
defer server.Close()
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := fetchManifestStruct(ctx, "repo", "tag", searchConf,
"", "")
So(err, ShouldNotBeNil)
})
Convey("makeGETRequest manifest error, context is not done", func() {
server := StartTestHTTPServer(HTTPRoutes{}, port)
defer server.Close()
_, err := fetchManifestStruct(context.Background(), "repo", "tag", searchConf,
"", "")
So(err, ShouldNotBeNil)
})
Convey("makeGETRequest config error, context is not done", func() {
server := StartTestHTTPServer(HTTPRoutes{
{
Route: "/v2/{name}/manifests/{reference}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
_, err := w.Write([]byte(`{"config":{"digest":"digest","size":0}}`))
if err != nil {
return
}
},
AllowedMethods: []string{http.MethodGet},
},
}, port)
defer server.Close()
_, err := fetchManifestStruct(context.Background(), "repo", "tag", searchConf,
"", "")
So(err, ShouldNotBeNil)
})
Convey("Platforms on config", func() {
server := StartTestHTTPServer(HTTPRoutes{
{
Route: "/v2/{name}/manifests/{reference}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
_, err := w.Write([]byte(`
{
"config":{
"digest":"digest",
"size":0,
"platform" : {
"os": "",
"architecture": "",
"variant": ""
}
}
}
`))
if err != nil {
return
}
},
AllowedMethods: []string{http.MethodGet},
},
{
Route: "/v2/{name}/blobs/{digest}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
_, err := w.Write([]byte(`
{
"architecture": "arch",
"os": "os",
"variant": "var"
}
`))
if err != nil {
return
}
},
AllowedMethods: []string{http.MethodGet},
},
}, port)
defer server.Close()
_, err := fetchManifestStruct(context.Background(), "repo", "tag", searchConf,
"", "")
So(err, ShouldBeNil)
})
Convey("isNotationSigned error", func() {
isSigned := isNotationSigned(context.Background(), "repo", "digest", searchConf,
"", "")
So(isSigned, ShouldBeFalse)
})
Convey("fetchImageIndexStruct no errors", func() {
server := StartTestHTTPServer(HTTPRoutes{
{
Route: "/v2/{name}/manifests/{reference}",
HandlerFunc: func(writer http.ResponseWriter, req *http.Request) {
vars := mux.Vars(req)
if vars["reference"] == "indexRef" {
_, err := writer.Write([]byte(`
{
"manifests": [
{
"digest": "manifestRef",
"platform": {
"architecture": "arch",
"os": "os",
"variant": "var"
}
}
]
}
`))
if err != nil {
return
}
} else if vars["reference"] == "manifestRef" {
_, err := writer.Write([]byte(`
{
"config":{
"digest":"digest",
"size":0
}
}
`))
if err != nil {
return
}
}
},
AllowedMethods: []string{http.MethodGet},
},
{
Route: "/v2/{name}/blobs/{digest}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
_, err := w.Write([]byte(`{}`))
if err != nil {
return
}
},
AllowedMethods: []string{http.MethodGet},
},
}, port)
defer server.Close()
URL := baseURL + "/v2/repo/manifests/indexRef"
imageStruct, err := fetchImageIndexStruct(context.Background(), &httpJob{
url: URL,
username: "",
password: "",
imageName: "repo",
tagName: "tag",
config: searchConf,
})
So(err, ShouldBeNil)
So(imageStruct, ShouldNotBeNil)
})
Convey("fetchImageIndexStruct makeGETRequest errors context done", func() {
server := StartTestHTTPServer(HTTPRoutes{}, port)
defer server.Close()
ctx, cancel := context.WithCancel(context.Background())
cancel()
URL := baseURL + "/v2/repo/manifests/indexRef"
imageStruct, err := fetchImageIndexStruct(ctx, &httpJob{
url: URL,
username: "",
password: "",
imageName: "repo",
tagName: "tag",
config: searchConf,
})
So(err, ShouldNotBeNil)
So(imageStruct, ShouldBeNil)
})
Convey("fetchImageIndexStruct makeGETRequest errors context not done", func() {
server := StartTestHTTPServer(HTTPRoutes{}, port)
defer server.Close()
URL := baseURL + "/v2/repo/manifests/indexRef"
imageStruct, err := fetchImageIndexStruct(context.Background(), &httpJob{
url: URL,
username: "",
password: "",
imageName: "repo",
tagName: "tag",
config: searchConf,
})
So(err, ShouldNotBeNil)
So(imageStruct, ShouldBeNil)
})
})
}
func TestDoJobErrors(t *testing.T) {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
searchConf := getDefaultSearchConf(baseURL)
reqPool := &requestsPool{
jobs: make(chan *httpJob),
done: make(chan struct{}),
wtgrp: &sync.WaitGroup{},
outputCh: make(chan stringResult),
}
Convey("Do Job errors", t, func() {
reqPool.wtgrp.Add(1)
Convey("Do Job makeHEADRequest error context done", func() {
server := StartTestHTTPServer(HTTPRoutes{}, port)
defer server.Close()
URL := baseURL + "/v2/repo/manifests/manifestRef"
ctx, cancel := context.WithCancel(context.Background())
cancel()
reqPool.doJob(ctx, &httpJob{
url: URL,
username: "",
password: "",
imageName: "",
tagName: "",
config: searchConf,
})
})
Convey("Do Job makeHEADRequest error context not done", func() {
server := StartTestHTTPServer(HTTPRoutes{}, port)
defer server.Close()
URL := baseURL + "/v2/repo/manifests/manifestRef"
ctx := context.Background()
go reqPool.doJob(ctx, &httpJob{
url: URL,
username: "",
password: "",
imageName: "",
tagName: "",
config: searchConf,
})
result := <-reqPool.outputCh
So(result.Err, ShouldNotBeNil)
So(result.StrValue, ShouldResemble, "")
})
Convey("Do Job fetchManifestStruct errors context canceled", func() {
server := StartTestHTTPServer(HTTPRoutes{
{
Route: "/v2/{name}/manifests/{reference}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Content-Type", ispec.MediaTypeImageManifest)
_, err := w.Write([]byte(""))
if err != nil {
return
}
},
AllowedMethods: []string{http.MethodHead},
},
}, port)
defer server.Close()
URL := baseURL + "/v2/repo/manifests/manifestRef"
ctx, cancel := context.WithCancel(context.Background())
cancel()
// context not canceled
reqPool.doJob(ctx, &httpJob{
url: URL,
username: "",
password: "",
imageName: "",
tagName: "",
config: searchConf,
})
})
Convey("Do Job fetchManifestStruct errors context not canceled", func() {
server := StartTestHTTPServer(HTTPRoutes{
{
Route: "/v2/{name}/manifests/{reference}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Content-Type", ispec.MediaTypeImageManifest)
_, err := w.Write([]byte(""))
if err != nil {
return
}
},
AllowedMethods: []string{http.MethodHead},
},
}, port)
defer server.Close()
URL := baseURL + "/v2/repo/manifests/manifestRef"
ctx := context.Background()
go reqPool.doJob(ctx, &httpJob{
url: URL,
username: "",
password: "",
imageName: "",
tagName: "",
config: searchConf,
})
result := <-reqPool.outputCh
So(result.Err, ShouldNotBeNil)
So(result.StrValue, ShouldResemble, "")
})
Convey("Do Job fetchIndexStruct errors context canceled", func() {
server := StartTestHTTPServer(HTTPRoutes{
{
Route: "/v2/{name}/manifests/{reference}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Content-Type", ispec.MediaTypeImageIndex)
_, err := w.Write([]byte(""))
if err != nil {
return
}
},
AllowedMethods: []string{http.MethodHead},
},
}, port)
defer server.Close()
URL := baseURL + "/v2/repo/manifests/indexRef"
ctx, cancel := context.WithCancel(context.Background())
cancel()
// context not canceled
reqPool.doJob(ctx, &httpJob{
url: URL,
username: "",
password: "",
imageName: "",
tagName: "",
config: searchConf,
})
})
Convey("Do Job fetchIndexStruct errors context not canceled", func() {
server := StartTestHTTPServer(HTTPRoutes{
{
Route: "/v2/{name}/manifests/{reference}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Content-Type", ispec.MediaTypeImageIndex)
_, err := w.Write([]byte(""))
if err != nil {
return
}
},
AllowedMethods: []string{http.MethodHead},
},
}, port)
defer server.Close()
URL := baseURL + "/v2/repo/manifests/indexRef"
ctx := context.Background()
go reqPool.doJob(ctx, &httpJob{
url: URL,
username: "",
password: "",
imageName: "",
tagName: "",
config: searchConf,
})
result := <-reqPool.outputCh
So(result.Err, ShouldNotBeNil)
So(result.StrValue, ShouldResemble, "")
})
Convey("Do Job fetchIndexStruct not supported content type", func() {
server := StartTestHTTPServer(HTTPRoutes{
{
Route: "/v2/{name}/manifests/{reference}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Content-Type", "some-media-type")
_, err := w.Write([]byte(""))
if err != nil {
return
}
},
AllowedMethods: []string{http.MethodHead},
},
}, port)
defer server.Close()
URL := baseURL + "/v2/repo/manifests/indexRef"
ctx := context.Background()
reqPool.doJob(ctx, &httpJob{
url: URL,
username: "",
password: "",
imageName: "",
tagName: "",
config: searchConf,
})
})
Convey("Media type is MediaTypeImageIndex image.string erorrs", func() {
server := StartTestHTTPServer(HTTPRoutes{
{
Route: "/v2/{name}/manifests/{reference}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Content-Type", ispec.MediaTypeImageIndex)
_, err := w.Write([]byte(""))
if err != nil {
return
}
},
AllowedMethods: []string{http.MethodHead},
},
{
Route: "/v2/{name}/manifests/{reference}",
HandlerFunc: func(writer http.ResponseWriter, req *http.Request) {
vars := mux.Vars(req)
if vars["reference"] == "indexRef" {
_, err := writer.Write([]byte(`{"manifests": [{"digest": "manifestRef"}]}`))
if err != nil {
return
}
}
if vars["reference"] == "manifestRef" {
_, err := writer.Write([]byte(`{"config": {"digest": "confDigest"}}`))
if err != nil {
return
}
}
},
AllowedMethods: []string{http.MethodGet},
},
{
Route: "/v2/{name}/blobs/{digest}",
HandlerFunc: func(w http.ResponseWriter, r *http.Request) {
_, err := w.Write([]byte(`{}`))
if err != nil {
return
}
},
AllowedMethods: []string{http.MethodGet},
},
}, port)
defer server.Close()
URL := baseURL + "/v2/repo/manifests/indexRef"
go reqPool.doJob(context.Background(), &httpJob{
url: URL,
username: "",
password: "",
imageName: "repo",
tagName: "indexRef",
config: searchConf,
})
result := <-reqPool.outputCh
So(result.Err, ShouldNotBeNil)
So(result.StrValue, ShouldResemble, "")
})
})
}

55
pkg/cli/cve_cmd_test.go
View file

@ -179,7 +179,8 @@ func TestSearchCVECmd(t *testing.T) {
So(err, ShouldBeNil)
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
So(strings.TrimSpace(str), ShouldEqual, "IMAGE NAME TAG DIGEST SIGNED SIZE dummyImageName tag 6e2f80bf false 123kB")
So(strings.TrimSpace(str), ShouldEqual,
"IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE dummyImageName tag 6e2f80bf os/arch false 123kB")
})
Convey("Test CVE by name and CVE ID - using shorthand", t, func() {
@ -195,7 +196,8 @@ func TestSearchCVECmd(t *testing.T) {
So(err, ShouldBeNil)
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
So(strings.TrimSpace(str), ShouldEqual, "IMAGE NAME TAG DIGEST SIGNED SIZE dummyImageName tag 6e2f80bf false 123kB")
So(strings.TrimSpace(str), ShouldEqual,
"IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE dummyImageName tag 6e2f80bf os/arch false 123kB")
})
Convey("Test CVE by image name - in text format", t, func() {
@ -278,7 +280,7 @@ func TestSearchCVECmd(t *testing.T) {
err := cveCmd.Execute()
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
So(strings.TrimSpace(str), ShouldEqual, "IMAGE NAME TAG DIGEST SIGNED SIZE anImage tag 6e2f80bf false 123kB")
So(strings.TrimSpace(str), ShouldEqual, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE anImage tag 6e2f80bf os/arch false 123kB") //nolint:lll
So(err, ShouldBeNil)
})
@ -323,7 +325,7 @@ func TestSearchCVECmd(t *testing.T) {
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
So(err, ShouldBeNil)
So(strings.TrimSpace(str), ShouldEqual, "IMAGE NAME TAG DIGEST SIGNED SIZE fixedImage tag 6e2f80bf false 123kB")
So(strings.TrimSpace(str), ShouldEqual, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE fixedImage tag 6e2f80bf os/arch false 123kB") //nolint:lll
})
Convey("Test fixed tags by and image name CVE ID - invalid image name", t, func() {
@ -625,8 +627,8 @@ func TestServerCVEResponse(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
str = strings.TrimSpace(str)
So(err, ShouldBeNil)
So(str, ShouldEqual, "IMAGE NAME TAG DIGEST SIGNED SIZE zot-cve-test 0.0.1 "+
test.GetTestBlobDigest("zot-cve-test", "manifest").Encoded()[:8]+" false 75MB")
So(str, ShouldEqual, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE zot-cve-test 0.0.1 "+
test.GetTestBlobDigest("zot-cve-test", "manifest").Encoded()[:8]+" N/A false 75MB")
})
Convey("Test images by CVE ID - GQL - invalid CVE ID", t, func() {
@ -643,7 +645,7 @@ func TestServerCVEResponse(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
str = strings.TrimSpace(str)
So(err, ShouldBeNil)
So(str, ShouldNotContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(str, ShouldNotContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
})
Convey("Test images by CVE ID - GQL - invalid output format", t, func() {
@ -691,7 +693,7 @@ func TestServerCVEResponse(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
str = strings.TrimSpace(str)
So(err, ShouldBeNil)
So(strings.TrimSpace(str), ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(strings.TrimSpace(str), ShouldContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
})
Convey("Test fixed tags by image name and CVE ID - GQL - random image", t, func() {
@ -708,7 +710,7 @@ func TestServerCVEResponse(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
str = strings.TrimSpace(str)
So(err, ShouldNotBeNil)
So(strings.TrimSpace(str), ShouldNotContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(strings.TrimSpace(str), ShouldNotContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
})
Convey("Test fixed tags by image name and CVE ID - GQL - invalid image", t, func() {
@ -725,7 +727,7 @@ func TestServerCVEResponse(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
str = strings.TrimSpace(str)
So(err, ShouldNotBeNil)
So(strings.TrimSpace(str), ShouldNotContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(strings.TrimSpace(str), ShouldNotContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
})
Convey("Test CVE by name and CVE ID - GQL - positive", t, func() {
@ -741,8 +743,8 @@ func TestServerCVEResponse(t *testing.T) {
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
So(err, ShouldBeNil)
So(strings.TrimSpace(str), ShouldEqual, "IMAGE NAME TAG DIGEST SIGNED SIZE zot-cve-test 0.0.1 "+
test.GetTestBlobDigest("zot-cve-test", "manifest").Encoded()[:8]+" false 75MB")
So(strings.TrimSpace(str), ShouldEqual, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE zot-cve-test 0.0.1 "+
test.GetTestBlobDigest("zot-cve-test", "manifest").Encoded()[:8]+" N/A false 75MB")
})
Convey("Test CVE by name and CVE ID - GQL - invalid name and CVE ID", t, func() {
@ -820,8 +822,8 @@ func TestServerCVEResponse(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
str = strings.TrimSpace(str)
So(err, ShouldBeNil)
So(str, ShouldEqual, "IMAGE NAME TAG DIGEST SIGNED SIZE zot-cve-test 0.0.1 "+
test.GetTestBlobDigest("zot-cve-test", "manifest").Encoded()[:8]+" false 75MB")
So(str, ShouldEqual, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE zot-cve-test 0.0.1 "+
test.GetTestBlobDigest("zot-cve-test", "manifest").Encoded()[:8]+" linux/amd64 false 75MB")
})
Convey("Test images by CVE ID - invalid CVE ID", t, func() {
@ -838,7 +840,7 @@ func TestServerCVEResponse(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
str = strings.TrimSpace(str)
So(err, ShouldBeNil)
So(str, ShouldNotContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(str, ShouldNotContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
})
Convey("Test fixed tags by and image name CVE ID - positive", t, func() {
@ -872,7 +874,7 @@ func TestServerCVEResponse(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
str = strings.TrimSpace(str)
So(err, ShouldBeNil)
So(strings.TrimSpace(str), ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(strings.TrimSpace(str), ShouldContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
})
Convey("Test fixed tags by and image name CVE ID - invalid image", t, func() {
@ -889,7 +891,7 @@ func TestServerCVEResponse(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
str = strings.TrimSpace(str)
So(err, ShouldNotBeNil)
So(strings.TrimSpace(str), ShouldNotContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(strings.TrimSpace(str), ShouldNotContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
})
Convey("Test CVE by name and CVE ID - positive", t, func() {
@ -905,8 +907,8 @@ func TestServerCVEResponse(t *testing.T) {
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
So(err, ShouldBeNil)
So(strings.TrimSpace(str), ShouldEqual, "IMAGE NAME TAG DIGEST SIGNED SIZE zot-cve-test 0.0.1 "+
test.GetTestBlobDigest("zot-cve-test", "manifest").Encoded()[:8]+" false 75MB")
So(strings.TrimSpace(str), ShouldEqual, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE zot-cve-test 0.0.1 "+
test.GetTestBlobDigest("zot-cve-test", "manifest").Encoded()[:8]+" linux/amd64 false 75MB")
})
Convey("Test CVE by name and CVE ID - invalid name and CVE ID", t, func() {
@ -922,7 +924,7 @@ func TestServerCVEResponse(t *testing.T) {
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
So(err, ShouldBeNil)
So(strings.TrimSpace(str), ShouldNotContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(strings.TrimSpace(str), ShouldNotContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
})
}
@ -1082,17 +1084,10 @@ func getMockCveInfo(repoDB repodb.RepoDB, log log.Logger) cveinfo.CveInfo {
CompareSeveritiesFn: func(severity1, severity2 string) int {
return severities[severity2] - severities[severity1]
},
IsImageFormatScannableFn: func(image string) (bool, error) {
IsImageFormatScannableFn: func(repo string, reference string) (bool, error) {
// Almost same logic compared to actual Trivy specific implementation
var imageDir string
var inputTag string
if strings.Contains(image, ":") {
imageDir, inputTag, _ = strings.Cut(image, ":")
} else {
imageDir = image
}
imageDir := repo
inputTag := reference
repoMeta, err := repoDB.GetRepoMeta(imageDir)
if err != nil {

273
pkg/cli/image_cmd_test.go
View file

@ -185,7 +185,8 @@ func TestSearchImageCmd(t *testing.T) {
err := cmd.Execute()
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
So(strings.TrimSpace(str), ShouldEqual, "IMAGE NAME TAG DIGEST SIGNED SIZE dummyImageName tag 6e2f80bf false 123kB")
So(strings.TrimSpace(str), ShouldEqual,
"IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE dummyImageName tag 6e2f80bf os/arch false 123kB")
So(err, ShouldBeNil)
})
@ -201,7 +202,8 @@ func TestSearchImageCmd(t *testing.T) {
err := imageCmd.Execute()
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
So(strings.TrimSpace(str), ShouldEqual, "IMAGE NAME TAG DIGEST SIGNED SIZE dummyImageName tag 6e2f80bf false 123kB")
So(strings.TrimSpace(str), ShouldEqual,
"IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE dummyImageName tag 6e2f80bf os/arch false 123kB")
So(err, ShouldBeNil)
Convey("using shorthand", func() {
args := []string{"imagetest", "-n", "dummyImageName", "--url", "someUrlImage"}
@ -216,7 +218,8 @@ func TestSearchImageCmd(t *testing.T) {
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
So(strings.TrimSpace(str), ShouldEqual, "IMAGE NAME TAG DIGEST SIGNED SIZE dummyImageName tag 6e2f80bf false 123kB")
So(strings.TrimSpace(str), ShouldEqual,
"IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE dummyImageName tag 6e2f80bf os/arch false 123kB")
So(err, ShouldBeNil)
})
})
@ -233,7 +236,8 @@ func TestSearchImageCmd(t *testing.T) {
err := imageCmd.Execute()
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
So(strings.TrimSpace(str), ShouldEqual, "IMAGE NAME TAG DIGEST SIGNED SIZE anImage tag 6e2f80bf false 123kB")
So(strings.TrimSpace(str), ShouldEqual,
"IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE anImage tag 6e2f80bf os/arch false 123kB")
So(err, ShouldBeNil)
Convey("invalid URL format", func() {
@ -282,10 +286,10 @@ func TestSignature(t *testing.T) {
repoName := "repo7"
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "test:1.0",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "test:1.0",
}, url, repoName)
So(err, ShouldBeNil)
@ -326,8 +330,8 @@ func TestSignature(t *testing.T) {
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:1.0 6742241d true 447B")
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:1.0 6742241d linux/amd64 true 447B")
t.Log("Test getting all images using rest calls to get catalog and individual manifests")
cmd = MockNewImageCommand(new(searchService))
@ -339,8 +343,8 @@ func TestSignature(t *testing.T) {
So(err, ShouldBeNil)
str = space.ReplaceAllString(buff.String(), " ")
actual = strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:1.0 6742241d true 447B")
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:1.0 6742241d linux/amd64 true 447B")
err = os.Chdir(currentWorkingDir)
So(err, ShouldBeNil)
@ -374,10 +378,10 @@ func TestSignature(t *testing.T) {
repoName := "repo7"
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "0.0.1",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "0.0.1",
}, url, repoName)
So(err, ShouldBeNil)
@ -403,8 +407,8 @@ func TestSignature(t *testing.T) {
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 0.0.1 6742241d true 447B")
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 0.0.1 6742241d linux/amd64 true 447B")
t.Log("Test getting all images using rest calls to get catalog and individual manifests")
cmd = MockNewImageCommand(new(searchService))
@ -416,8 +420,8 @@ func TestSignature(t *testing.T) {
So(err, ShouldBeNil)
str = space.ReplaceAllString(buff.String(), " ")
actual = strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 0.0.1 6742241d true 447B")
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 0.0.1 6742241d linux/amd64 true 447B")
err = os.Chdir(currentWorkingDir)
So(err, ShouldBeNil)
@ -465,8 +469,8 @@ func TestDerivedImageList(t *testing.T) {
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:1.0 2694fdb0 false 824B")
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:1.0 2694fdb0 N/A false 824B")
})
Convey("Test derived images list fails", func() {
@ -538,8 +542,8 @@ func TestBaseImageList(t *testing.T) {
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 3fc80493 false 494B")
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 3fc80493 N/A false 494B")
})
Convey("Test base images list fail", func() {
@ -727,7 +731,8 @@ func TestOutputFormat(t *testing.T) {
err := cmd.Execute()
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
So(strings.TrimSpace(str), ShouldEqual, "IMAGE NAME TAG DIGEST SIGNED SIZE dummyImageName tag 6e2f80bf false 123kB")
So(strings.TrimSpace(str), ShouldEqual,
"IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE dummyImageName tag 6e2f80bf os/arch false 123kB")
So(err, ShouldBeNil)
})
@ -746,10 +751,11 @@ func TestOutputFormat(t *testing.T) {
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
So(strings.TrimSpace(str), ShouldEqual, `{ "repoName": "dummyImageName", "tag": "tag", `+
`"configDigest": "sha256:4c10985c40365538426f2ba8cf0c21384a7769be502a550dcc0601b3736625e0", `+
`"Manifests": [ { "configDigest": "sha256:4c10985c40365538426f2ba8cf0c21384a7769be502a550dcc0601b3736625e0", `+
`"digest": "sha256:6e2f80bf9cfaabad474fbaf8ad68fdb652f776ea80b63492ecca404e5f6446a6", `+
`"layers": [ { "size": "0", `+
`"digest": "sha256:c122a146f0d02349be211bb95cc2530f4a5793f96edbdfa00860f741e5d8c0e6" } ], `+
`"layers": [ { "size": "0", "digest": "sha256:c122a146f0d02349be211bb95cc2530f4a5793f96edbdfa00860f741e5d8c0e6" } ], `+ //nolint:lll
`"platform": { "os": "os", "arch": "arch", "variant": "" }, `+
`"size": "123445", "isSigned": false } ], `+
`"size": "123445", "isSigned": false }`)
So(err, ShouldBeNil)
})
@ -770,9 +776,12 @@ func TestOutputFormat(t *testing.T) {
strings.TrimSpace(str),
ShouldEqual,
`reponame: dummyImageName tag: tag `+
`manifests: - `+
`configdigest: sha256:4c10985c40365538426f2ba8cf0c21384a7769be502a550dcc0601b3736625e0 `+
`digest: sha256:6e2f80bf9cfaabad474fbaf8ad68fdb652f776ea80b63492ecca404e5f6446a6 `+
`layers: - size: 0 digest: sha256:c122a146f0d02349be211bb95cc2530f4a5793f96edbdfa00860f741e5d8c0e6 `+
`platform: os: os arch: arch variant: "" `+
`size: "123445" issigned: false `+
`size: "123445" issigned: false`,
)
So(err, ShouldBeNil)
@ -796,9 +805,12 @@ func TestOutputFormat(t *testing.T) {
strings.TrimSpace(str),
ShouldEqual,
`reponame: dummyImageName tag: tag `+
`manifests: - `+
`configdigest: sha256:4c10985c40365538426f2ba8cf0c21384a7769be502a550dcc0601b3736625e0 `+
`digest: sha256:6e2f80bf9cfaabad474fbaf8ad68fdb652f776ea80b63492ecca404e5f6446a6 `+
`layers: - size: 0 digest: sha256:c122a146f0d02349be211bb95cc2530f4a5793f96edbdfa00860f741e5d8c0e6 `+
`platform: os: os arch: arch variant: "" `+
`size: "123445" issigned: false `+
`size: "123445" issigned: false`,
)
So(err, ShouldBeNil)
@ -855,9 +867,9 @@ func TestServerResponseGQL(t *testing.T) {
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 linux/amd64 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 linux/amd64 false 492B")
Convey("Test all images invalid output format", func() {
args := []string{"imagetest", "-o", "random"}
configPath := makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"imagetest","url":"%s","showspinner":false}]}`, url))
@ -888,14 +900,14 @@ func TestServerResponseGQL(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
// Actual cli output should be something similar to (order of images may differ):
// IMAGE NAME TAG DIGEST CONFIG SIGNED LAYERS SIZE
// repo7 test:2.0 a0ca253b b8781e88 false 492B
// b8781e88 15B
// repo7 test:1.0 a0ca253b b8781e88 false 492B
// b8781e88 15B
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST CONFIG SIGNED LAYERS SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 3a1d2d0c false 492B b8781e88 15B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 3a1d2d0c false 492B b8781e88 15B")
// IMAGE NAME TAG DIGEST CONFIG OS/ARCH SIGNED LAYERS SIZE
// repo7 test:2.0 a0ca253b b8781e88 linux/amd64 false 492B
// b8781e88 15B
// repo7 test:1.0 a0ca253b b8781e88 linux/amd64 false 492B
// b8781e88 15B
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST CONFIG OS/ARCH SIGNED LAYERS SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 3a1d2d0c linux/amd64 false 492B b8781e88 15B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 3a1d2d0c linux/amd64 false 492B b8781e88 15B")
})
Convey("Test all images with debug flag", func() {
@ -913,9 +925,9 @@ func TestServerResponseGQL(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "GET")
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 linux/amd64 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 linux/amd64 false 492B")
})
Convey("Test image by name config url", func() {
@ -932,9 +944,9 @@ func TestServerResponseGQL(t *testing.T) {
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 linux/amd64 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 linux/amd64 false 492B")
Convey("with shorthand", func() {
args := []string{"imagetest", "-n", "repo7"}
@ -950,9 +962,9 @@ func TestServerResponseGQL(t *testing.T) {
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 linux/amd64 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 linux/amd64 false 492B")
})
Convey("invalid output format", func() {
@ -985,12 +997,12 @@ func TestServerResponseGQL(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
// Actual cli output should be something similar to (order of images may differ):
// IMAGE NAME TAG DIGEST SIZE
// repo7 test:2.0 a0ca253b 15B
// repo7 test:1.0 a0ca253b 15B
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B")
// IMAGE NAME TAG DIGEST OS/ARCH SIZE
// repo7 test:2.0 a0ca253b N/A 15B
// repo7 test:1.0 a0ca253b N/A 15B
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 N/A false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 N/A false 492B")
Convey("with shorthand", func() {
args := []string{"imagetest", "-d", "883fc0c5"}
@ -1006,9 +1018,9 @@ func TestServerResponseGQL(t *testing.T) {
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 N/A false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 N/A false 492B")
})
Convey("nonexistent digest", func() {
@ -1116,9 +1128,9 @@ func TestServerResponse(t *testing.T) {
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 linux/amd64 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 linux/amd64 false 492B")
})
Convey("Test all images verbose", func() {
@ -1136,14 +1148,14 @@ func TestServerResponse(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
// Actual cli output should be something similar to (order of images may differ):
// IMAGE NAME TAG DIGEST CONFIG SIGNED LAYERS SIZE
// repo7 test:2.0 a0ca253b b8781e88 false 492B
// b8781e88 15B
// repo7 test:1.0 a0ca253b b8781e88 false 492B
// b8781e88 15B
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST CONFIG SIGNED LAYERS SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 3a1d2d0c false 492B b8781e88 15B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 3a1d2d0c false 492B b8781e88 15B")
// IMAGE NAME TAG DIGEST CONFIG OS/ARCH SIGNED LAYERS SIZE
// repo7 test:2.0 a0ca253b b8781e88 linux/amd64 false 492B
// linux/amd64 b8781e88 15B
// repo7 test:1.0 a0ca253b b8781e88 linux/amd64 false 492B
// linux/amd64 b8781e88 15B
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST CONFIG OS/ARCH SIGNED LAYERS SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 3a1d2d0c linux/amd64 false 492B b8781e88 15B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 3a1d2d0c linux/amd64 false 492B b8781e88 15B")
})
Convey("Test image by name", func() {
@ -1160,9 +1172,9 @@ func TestServerResponse(t *testing.T) {
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 linux/amd64 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 linux/amd64 false 492B")
})
Convey("Test image by digest", func() {
@ -1180,12 +1192,12 @@ func TestServerResponse(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
// Actual cli output should be something similar to (order of images may differ):
// IMAGE NAME TAG DIGEST SIZE
// repo7 test:2.0 a0ca253b 492B
// repo7 test:1.0 a0ca253b 492B
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B")
// IMAGE NAME TAG DIGEST OS/ARCH SIZE
// repo7 test:2.0 a0ca253b linux/amd64 492B
// repo7 test:1.0 a0ca253b linux/amd64 492B
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST OS/ARCH SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 linux/amd64 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 linux/amd64 false 492B")
Convey("nonexistent digest", func() {
args := []string{"imagetest", "--digest", "d1g35t"}
@ -1538,12 +1550,17 @@ func (service mockService) getDerivedImageListGQL(ctx context.Context, config se
imageListGQLResponse := &imageListStructForDerivedImagesGQL{}
imageListGQLResponse.Data.Results = []imageStruct{
{
RepoName: "dummyImageName",
Tag: "tag",
Digest: godigest.FromString("Digest").String(),
ConfigDigest: godigest.FromString("ConfigDigest").String(),
Size: "123445",
Layers: []layer{{Digest: godigest.FromString("LayerDigest").String()}},
RepoName: "dummyImageName",
Tag: "tag",
Manifests: []manifestStruct{
{
Digest: godigest.FromString("Digest").String(),
ConfigDigest: godigest.FromString("ConfigDigest").String(),
Size: "123445",
Layers: []layer{{Digest: godigest.FromString("LayerDigest").String()}},
},
},
Size: "123445",
},
}
@ -1556,12 +1573,17 @@ func (service mockService) getBaseImageListGQL(ctx context.Context, config searc
imageListGQLResponse := &imageListStructForBaseImagesGQL{}
imageListGQLResponse.Data.Results = []imageStruct{
{
RepoName: "dummyImageName",
Tag: "tag",
Digest: godigest.FromString("Digest").String(),
ConfigDigest: godigest.FromString("ConfigDigest").String(),
Size: "123445",
Layers: []layer{{Digest: godigest.FromString("LayerDigest").String()}},
RepoName: "dummyImageName",
Tag: "tag",
Manifests: []manifestStruct{
{
Digest: godigest.FromString("Digest").String(),
ConfigDigest: godigest.FromString("ConfigDigest").String(),
Size: "123445",
Layers: []layer{{Digest: godigest.FromString("LayerDigest").String()}},
},
},
Size: "123445",
},
}
@ -1574,12 +1596,17 @@ func (service mockService) getImagesGQL(ctx context.Context, config searchConfig
imageListGQLResponse := &imageListStructGQL{}
imageListGQLResponse.Data.Results = []imageStruct{
{
RepoName: "dummyImageName",
Tag: "tag",
Digest: godigest.FromString("Digest").String(),
ConfigDigest: godigest.FromString("ConfigDigest").String(),
Size: "123445",
Layers: []layer{{Digest: godigest.FromString("LayerDigest").String()}},
RepoName: "dummyImageName",
Tag: "tag",
Manifests: []manifestStruct{
{
Digest: godigest.FromString("Digest").String(),
ConfigDigest: godigest.FromString("ConfigDigest").String(),
Size: "123445",
Layers: []layer{{Digest: godigest.FromString("LayerDigest").String()}},
},
},
Size: "123445",
},
}
@ -1592,12 +1619,17 @@ func (service mockService) getImagesByDigestGQL(ctx context.Context, config sear
imageListGQLResponse := &imageListStructForDigestGQL{}
imageListGQLResponse.Data.Results = []imageStruct{
{
RepoName: "randomimageName",
Tag: "tag",
Digest: godigest.FromString("Digest").String(),
ConfigDigest: godigest.FromString("ConfigDigest").String(),
Size: "123445",
Layers: []layer{{Digest: godigest.FromString("LayerDigest").String()}},
RepoName: "randomimageName",
Tag: "tag",
Manifests: []manifestStruct{
{
Digest: godigest.FromString("Digest").String(),
ConfigDigest: godigest.FromString("ConfigDigest").String(),
Layers: []layer{{Digest: godigest.FromString("LayerDigest").String()}},
Size: "123445",
},
},
Size: "123445",
},
}
@ -1691,10 +1723,15 @@ func (service mockService) getMockedImageByName(imageName string) imageStruct {
image := imageStruct{}
image.RepoName = imageName
image.Tag = "tag"
image.Digest = godigest.FromString("Digest").String()
image.ConfigDigest = godigest.FromString("ConfigDigest").String()
image.Manifests = []manifestStruct{
{
Digest: godigest.FromString("Digest").String(),
ConfigDigest: godigest.FromString("ConfigDigest").String(),
Layers: []layer{{Digest: godigest.FromString("LayerDigest").String()}},
Size: "123445",
},
}
image.Size = "123445"
image.Layers = []layer{{Digest: godigest.FromString("LayerDigest").String()}}
return image
}
@ -1708,12 +1745,18 @@ func (service mockService) getAllImages(ctx context.Context, config searchConfig
image := &imageStruct{}
image.RepoName = "randomimageName"
image.Tag = "tag"
image.Digest = godigest.FromString("Digest").String()
image.ConfigDigest = godigest.FromString("ConfigDigest").String()
image.Manifests = []manifestStruct{
{
Digest: godigest.FromString("Digest").String(),
ConfigDigest: godigest.FromString("ConfigDigest").String(),
Layers: []layer{{Digest: godigest.FromString("LayerDigest").String()}},
Size: "123445",
Platform: platform{Os: "os", Arch: "arch"},
},
}
image.Size = "123445"
image.Layers = []layer{{Digest: godigest.FromString("LayerDigest").String()}}
str, err := image.string(*config.outputFormat, len(image.RepoName), len(image.Tag))
str, err := image.string(*config.outputFormat, len(image.RepoName), len(image.Tag), len("os/Arch"))
if err != nil {
channel <- stringResult{"", err}
@ -1732,12 +1775,18 @@ func (service mockService) getImageByName(ctx context.Context, config searchConf
image := &imageStruct{}
image.RepoName = imageName
image.Tag = "tag"
image.Digest = godigest.FromString("Digest").String()
image.ConfigDigest = godigest.FromString("ConfigDigest").String()
image.Manifests = []manifestStruct{
{
Digest: godigest.FromString("Digest").String(),
ConfigDigest: godigest.FromString("ConfigDigest").String(),
Layers: []layer{{Digest: godigest.FromString("LayerDigest").String()}},
Size: "123445",
Platform: platform{Os: "os", Arch: "arch"},
},
}
image.Size = "123445"
image.Layers = []layer{{Digest: godigest.FromString("LayerDigest").String()}}
str, err := image.string(*config.outputFormat, len(image.RepoName), len(image.Tag))
str, err := image.string(*config.outputFormat, len(image.RepoName), len(image.Tag), len("os/Arch"))
if err != nil {
channel <- stringResult{"", err}

33
pkg/cli/searcher.go
View file

@ -359,7 +359,7 @@ func (search cveByImageSearcherGQL) search(config searchConfig) (bool, error) {
if len(cveList.Data.CVEListForImage.CVEList) > 0 &&
(*config.outputFormat == defaultOutoutFormat || *config.outputFormat == "") {
printCVETableHeader(&builder, *config.verbose, 0, 0)
printCVETableHeader(&builder, *config.verbose, 0, 0, 0)
fmt.Fprint(config.resultWriter, builder.String())
}
@ -596,7 +596,7 @@ func collectResults(config searchConfig, wg *sync.WaitGroup, imageErr chan strin
if !foundResult && (*config.outputFormat == defaultOutoutFormat || *config.outputFormat == "") {
var builder strings.Builder
printHeader(&builder, *config.verbose, 0, 0)
printHeader(&builder, *config.verbose, 0, 0, 0)
fmt.Fprint(config.resultWriter, builder.String())
}
@ -696,13 +696,14 @@ type stringResult struct {
Err error
}
type printHeader func(writer io.Writer, verbose bool, maxImageNameLen, maxTagLen int)
type printHeader func(writer io.Writer, verbose bool, maxImageNameLen, maxTagLen, maxPlatformLen int)
func printImageTableHeader(writer io.Writer, verbose bool, maxImageNameLen, maxTagLen int) {
func printImageTableHeader(writer io.Writer, verbose bool, maxImageNameLen, maxTagLen, maxPlatformLen int) {
table := getImageTableWriter(writer)
table.SetColMinWidth(colImageNameIndex, imageNameWidth)
table.SetColMinWidth(colTagIndex, tagWidth)
table.SetColMinWidth(colPlatformIndex, platformWidth)
table.SetColMinWidth(colDigestIndex, digestWidth)
table.SetColMinWidth(colSizeIndex, sizeWidth)
table.SetColMinWidth(colIsSignedIndex, isSignedWidth)
@ -712,7 +713,7 @@ func printImageTableHeader(writer io.Writer, verbose bool, maxImageNameLen, maxT
table.SetColMinWidth(colLayersIndex, layersWidth)
}
row := make([]string, 7) //nolint:gomnd
row := make([]string, 8) //nolint:gomnd
// adding spaces so that image name and tag columns are aligned
// in case the name/tag are fully shown and too long
@ -731,6 +732,13 @@ func printImageTableHeader(writer io.Writer, verbose bool, maxImageNameLen, maxT
row[colTagIndex] = "TAG"
}
if maxPlatformLen > len("OS/ARCH") {
offset = strings.Repeat(" ", maxPlatformLen-len("OS/ARCH"))
row[colPlatformIndex] = "OS/ARCH" + offset
} else {
row[colPlatformIndex] = "OS/ARCH"
}
row[colDigestIndex] = "DIGEST"
row[colSizeIndex] = "SIZE"
row[colIsSignedIndex] = "SIGNED"
@ -744,7 +752,7 @@ func printImageTableHeader(writer io.Writer, verbose bool, maxImageNameLen, maxT
table.Render()
}
func printCVETableHeader(writer io.Writer, verbose bool, maxImgLen, maxTagLen int) {
func printCVETableHeader(writer io.Writer, verbose bool, maxImgLen, maxTagLen, maxPlatformLen int) {
table := getCVETableWriter(writer)
row := make([]string, 3) //nolint:gomnd
row[colCVEIDIndex] = "ID"
@ -759,6 +767,7 @@ func printResult(config searchConfig, imageList []imageStruct) error {
var builder strings.Builder
maxImgNameLen := 0
maxTagLen := 0
maxPlatformLen := 0
if len(imageList) > 0 {
for i := range imageList {
@ -769,9 +778,17 @@ func printResult(config searchConfig, imageList []imageStruct) error {
if maxTagLen < len(imageList[i].Tag) {
maxTagLen = len(imageList[i].Tag)
}
for j := range imageList[i].Manifests {
platform := imageList[i].Manifests[j].Platform.Os + "/" + imageList[i].Manifests[j].Platform.Arch
if maxPlatformLen < len(platform) {
maxPlatformLen = len(platform)
}
}
}
printImageTableHeader(&builder, *config.verbose, maxImgNameLen, maxTagLen)
printImageTableHeader(&builder, *config.verbose, maxImgNameLen, maxTagLen, maxPlatformLen)
fmt.Fprint(config.resultWriter, builder.String())
}
@ -779,7 +796,7 @@ func printResult(config searchConfig, imageList []imageStruct) error {
img := imageList[i]
img.verbose = *config.verbose
out, err := img.string(*config.outputFormat, maxImgNameLen, maxTagLen)
out, err := img.string(*config.outputFormat, maxImgNameLen, maxTagLen, maxPlatformLen)
if err != nil {
return err
}

413
pkg/cli/service.go
View file

@ -74,9 +74,13 @@ func (service searchService) getDerivedImageListGQL(ctx context.Context, config
Results{
RepoName,
Tag,
Digest,
ConfigDigest,
Layers {Size Digest},
Manifests {
Digest,
ConfigDigest,
Layers {Size Digest},
LastUpdated,
Size
},
LastUpdated,
IsSigned,
Size
@ -103,9 +107,13 @@ func (service searchService) getBaseImageListGQL(ctx context.Context, config sea
Results{
RepoName,
Tag,
Digest,
ConfigDigest,
Layers {Size Digest},
Manifests {
Digest,
ConfigDigest,
Layers {Size Digest},
LastUpdated,
Size
},
LastUpdated,
IsSigned,
Size
@ -126,9 +134,23 @@ func (service searchService) getBaseImageListGQL(ctx context.Context, config sea
func (service searchService) getImagesGQL(ctx context.Context, config searchConfig, username, password string,
imageName string,
) (*imageListStructGQL, error) {
query := fmt.Sprintf(`{ImageList(repo: "%s") { Results {`+`
RepoName Tag Digest ConfigDigest Size Layers {Size Digest} IsSigned}}
}`,
query := fmt.Sprintf(`
{
ImageList(repo: "%s") {
Results {
RepoName Tag
Manifests {
Digest
ConfigDigest
Size
Platform {Os Arch}
Layers {Size Digest}
}
Size
IsSigned
}
}
}`,
imageName)
result := &imageListStructGQL{}
@ -144,9 +166,22 @@ func (service searchService) getImagesGQL(ctx context.Context, config searchConf
func (service searchService) getImagesByDigestGQL(ctx context.Context, config searchConfig, username, password string,
digest string,
) (*imageListStructForDigestGQL, error) {
query := fmt.Sprintf(`{ImageListForDigest(id: "%s") { Results{`+`
RepoName Tag Digest ConfigDigest Size Layers {Size Digest}}}
}`,
query := fmt.Sprintf(`
{
ImageListForDigest(id: "%s") {
Results {
RepoName Tag
Manifests {
Digest
ConfigDigest
Size
Layers {Size Digest}
}
Size
IsSigned
}
}
}`,
digest)
result := &imageListStructForDigestGQL{}
@ -162,9 +197,22 @@ func (service searchService) getImagesByDigestGQL(ctx context.Context, config se
func (service searchService) getImagesByCveIDGQL(ctx context.Context, config searchConfig, username,
password, cveID string,
) (*imagesForCve, error) {
query := fmt.Sprintf(`{ImageListForCVE(id: "%s") { Results {`+`
RepoName Tag Digest ConfigDigest Layers {Size Digest} Size}}
}`,
query := fmt.Sprintf(`
{
ImageListForCVE(id: "%s") {
Results {
RepoName Tag
Manifests {
Digest
ConfigDigest
Size
Layers {Size Digest}
}
Size
IsSigned
}
}
}`,
cveID)
result := &imagesForCve{}
@ -199,9 +247,21 @@ func (service searchService) getCveByImageGQL(ctx context.Context, config search
func (service searchService) getTagsForCVEGQL(ctx context.Context, config searchConfig,
username, password, imageName, cveID string,
) (*imagesForCve, error) {
query := fmt.Sprintf(`{ImageListForCVE(id: "%s") { Results {`+`
RepoName Tag Digest ConfigDigest Layers {Size Digest} Size}}
}`,
query := fmt.Sprintf(`
{
ImageListForCVE(id: "%s") {
Results {
RepoName Tag
Manifests {
Digest
ConfigDigest
Size
Layers {Size Digest}
}
Size
}
}
}`,
cveID)
result := &imagesForCve{}
@ -217,9 +277,21 @@ func (service searchService) getTagsForCVEGQL(ctx context.Context, config search
func (service searchService) getFixedTagsForCVEGQL(ctx context.Context, config searchConfig,
username, password, imageName, cveID string,
) (*fixedTags, error) {
query := fmt.Sprintf(`{ImageListWithCVEFixed(id: "%s", image: "%s") { Results {`+`
RepoName Tag Digest ConfigDigest Layers {Size Digest} Size}}
}`,
query := fmt.Sprintf(`
{
ImageListWithCVEFixed(id: "%s", image: "%s") {
Results {
RepoName Tag
Manifests {
Digest
ConfigDigest
Size
Layers {Size Digest}
}
Size
}
}
}`,
cveID, imageName)
result := &fixedTags{}
@ -349,10 +421,23 @@ func (service searchService) getImagesByCveID(ctx context.Context, config search
defer wtgrp.Done()
defer close(rch)
query := fmt.Sprintf(`{ImageListForCVE(id: "%s") { Results {`+`
RepoName Tag Digest ConfigDigest Layers {Size Digest} Size}}
}`,
query := fmt.Sprintf(
`{
ImageListForCVE(id: "%s") {
Results {
RepoName Tag
Manifests {
Digest
ConfigDigest
Size
Layers {Size Digest}
}
Size
}
}
}`,
cvid)
result := &imagesForCve{}
err := service.makeGraphQLQuery(ctx, config, username, password, query, result)
@ -402,10 +487,23 @@ func (service searchService) getImagesByDigest(ctx context.Context, config searc
defer wtgrp.Done()
defer close(rch)
query := fmt.Sprintf(`{ImageListForDigest(id: "%s") { Results {`+`
RepoName Tag Digest ConfigDigest Size Layers {Size Digest}}}
}`,
query := fmt.Sprintf(
`{
ImageListForDigest(id: "%s") {
Results {
RepoName Tag
Manifests {
Digest
ConfigDigest
Size
Layers {Size Digest}
}
Size
}
}
}`,
digest)
result := &imagesForDigest{}
err := service.makeGraphQLQuery(ctx, config, username, password, query, result)
@ -455,10 +553,23 @@ func (service searchService) getImageByNameAndCVEID(ctx context.Context, config
defer wtgrp.Done()
defer close(rch)
query := fmt.Sprintf(`{ImageListForCVE(id: "%s") { Results {`+`
RepoName Tag Digest ConfigDigest Size Layers {Size Digest}}}
}`,
query := fmt.Sprintf(
`{
ImageListForCVE(id: "%s") {
Results {
RepoName Tag
Manifests {
Digest
ConfigDigest
Size
Layers {Size Digest}
}
Size
}
}
}`,
cvid)
result := &imagesForCve{}
err := service.makeGraphQLQuery(ctx, config, username, password, query, result)
@ -566,10 +677,22 @@ func (service searchService) getFixedTagsForCVE(ctx context.Context, config sear
defer wtgrp.Done()
defer close(rch)
query := fmt.Sprintf(`{ImageListWithCVEFixed (id: "%s", image: "%s") { Results {`+`
RepoName Tag Digest ConfigDigest Layers {Size Digest} Size}}
}`,
cvid, imageName)
query := fmt.Sprintf(`
{
ImageListWithCVEFixed (id: "%s", image: "%s") {
Results {
RepoName Tag
Manifests {
Digest
ConfigDigest
Size
Layers {Size Digest}
}
Size
}
}
}`, cvid, imageName)
result := &fixedTags{}
err := service.makeGraphQLQuery(ctx, config, username, password, query, result)
@ -719,8 +842,6 @@ func addManifestCallToPool(ctx context.Context, config searchConfig, pool *reque
) {
defer wtgrp.Done()
resultManifest := manifestResponse{}
manifestEndpoint, err := combineServerAndEndpointURL(*config.servURL,
fmt.Sprintf("/v2/%s/manifests/%s", imageName, tagName))
if err != nil {
@ -730,14 +851,13 @@ func addManifestCallToPool(ctx context.Context, config searchConfig, pool *reque
rch <- stringResult{"", err}
}
job := manifestJob{
url: manifestEndpoint,
username: username,
imageName: imageName,
password: password,
tagName: tagName,
manifestResp: resultManifest,
config: config,
job := httpJob{
url: manifestEndpoint,
username: username,
imageName: imageName,
password: password,
tagName: tagName,
config: config,
}
wtgrp.Add(1)
@ -860,14 +980,27 @@ type PaginatedImagesResult struct {
}
type imageStruct struct {
RepoName string `json:"repoName"`
Tag string `json:"tag"`
ConfigDigest string `json:"configDigest"`
Digest string `json:"digest"`
Layers []layer `json:"layers"`
Size string `json:"size"`
verbose bool
IsSigned bool `json:"isSigned"`
RepoName string `json:"repoName"`
Tag string `json:"tag"`
Manifests []manifestStruct
Size string `json:"size"`
verbose bool
IsSigned bool `json:"isSigned"`
}
type manifestStruct struct {
ConfigDigest string `json:"configDigest"`
Digest string `json:"digest"`
Layers []layer `json:"layers"`
Platform platform `json:"platform"`
Size string `json:"size"`
IsSigned bool `json:"isSigned"`
}
type platform struct {
Os string `json:"os"`
Arch string `json:"arch"`
Variant string `json:"variant"`
}
type DerivedImageList struct {
@ -913,14 +1046,14 @@ type imagesForDigest struct {
}
type layer struct {
Size uint64 `json:"size,string"`
Size int64 `json:"size,string"`
Digest string `json:"digest"`
}
func (img imageStruct) string(format string, maxImgNameLen, maxTagLen int) (string, error) {
func (img imageStruct) string(format string, maxImgNameLen, maxTagLen, maxPlatformLen int) (string, error) {
switch strings.ToLower(format) {
case "", defaultOutoutFormat:
return img.stringPlainText(maxImgNameLen, maxTagLen)
return img.stringPlainText(maxImgNameLen, maxTagLen, maxPlatformLen)
case "json":
return img.stringJSON()
case "yml", "yaml":
@ -930,14 +1063,14 @@ func (img imageStruct) string(format string, maxImgNameLen, maxTagLen int) (stri
}
}
func (img imageStruct) stringPlainText(maxImgNameLen, maxTagLen int) (string, error) {
func (img imageStruct) stringPlainText(maxImgNameLen, maxTagLen, maxPlatformLen int) (string, error) {
var builder strings.Builder
table := getImageTableWriter(&builder)
table.SetColMinWidth(colImageNameIndex, maxImgNameLen)
table.SetColMinWidth(colTagIndex, maxTagLen)
table.SetColMinWidth(colPlatformIndex, platformWidth)
table.SetColMinWidth(colDigestIndex, digestWidth)
table.SetColMinWidth(colSizeIndex, sizeWidth)
table.SetColMinWidth(colIsSignedIndex, isSignedWidth)
@ -952,57 +1085,89 @@ func (img imageStruct) stringPlainText(maxImgNameLen, maxTagLen int) (string, er
imageName = img.RepoName
tagName = img.Tag
manifestDigest, err := godigest.Parse(img.Digest)
if err != nil {
return "", fmt.Errorf("error parsing manifest digest %s: %w", img.Digest, err)
if imageNameWidth > maxImgNameLen {
maxImgNameLen = imageNameWidth
}
configDigest, err := godigest.Parse(img.ConfigDigest)
if err != nil {
return "", fmt.Errorf("error parsing config digest %s: %w", img.ConfigDigest, err)
if tagWidth > maxTagLen {
maxTagLen = tagWidth
}
minifestDigestStr := ellipsize(manifestDigest.Encoded(), digestWidth, "")
configDigestStr := ellipsize(configDigest.Encoded(), configWidth, "")
imgSize, _ := strconv.ParseUint(img.Size, 10, 64)
size := ellipsize(strings.ReplaceAll(humanize.Bytes(imgSize), " ", ""), sizeWidth, ellipsis)
isSigned := img.IsSigned
row := make([]string, 7) //nolint:gomnd
row[colImageNameIndex] = imageName
row[colTagIndex] = tagName
row[colDigestIndex] = minifestDigestStr
row[colSizeIndex] = size
row[colIsSignedIndex] = strconv.FormatBool(isSigned)
if img.verbose {
row[colConfigIndex] = configDigestStr
row[colLayersIndex] = ""
// adding spaces so that image name and tag columns are aligned
// in case the name/tag are fully shown and too long
var offset string
if maxImgNameLen > len(imageName) {
offset = strings.Repeat(" ", maxImgNameLen-len(imageName))
imageName += offset
}
table.Append(row)
if maxTagLen > len(tagName) {
offset = strings.Repeat(" ", maxTagLen-len(tagName))
tagName += offset
}
if img.verbose {
for _, entry := range img.Layers {
layerSize := entry.Size
size := ellipsize(strings.ReplaceAll(humanize.Bytes(layerSize), " ", ""), sizeWidth, ellipsis)
for i := range img.Manifests {
manifestDigest, err := godigest.Parse(img.Manifests[i].Digest)
if err != nil {
return "", fmt.Errorf("error parsing manifest digest %s: %w", img.Manifests[i].Digest, err)
}
layerDigest, err := godigest.Parse(entry.Digest)
if err != nil {
return "", fmt.Errorf("error parsing layer digest %s: %w", entry.Digest, err)
configDigest, err := godigest.Parse(img.Manifests[i].ConfigDigest)
if err != nil {
return "", fmt.Errorf("error parsing config digest %s: %w", img.Manifests[i].ConfigDigest, err)
}
platform := getPlatformStr(img.Manifests[i].Platform)
if maxPlatformLen > len(platform) {
offset = strings.Repeat(" ", maxPlatformLen-len(platform))
platform += offset
}
minifestDigestStr := ellipsize(manifestDigest.Encoded(), digestWidth, "")
configDigestStr := ellipsize(configDigest.Encoded(), configWidth, "")
imgSize, _ := strconv.ParseUint(img.Manifests[i].Size, 10, 64)
size := ellipsize(strings.ReplaceAll(humanize.Bytes(imgSize), " ", ""), sizeWidth, ellipsis)
isSigned := img.IsSigned
row := make([]string, 8) //nolint:gomnd
row[colImageNameIndex] = imageName
row[colTagIndex] = tagName
row[colDigestIndex] = minifestDigestStr
row[colPlatformIndex] = platform
row[colSizeIndex] = size
row[colIsSignedIndex] = strconv.FormatBool(isSigned)
if img.verbose {
row[colConfigIndex] = configDigestStr
row[colLayersIndex] = ""
}
table.Append(row)
if img.verbose {
for _, entry := range img.Manifests[i].Layers {
layerSize := entry.Size
size := ellipsize(strings.ReplaceAll(humanize.Bytes(uint64(layerSize)), " ", ""), sizeWidth, ellipsis)
layerDigest, err := godigest.Parse(entry.Digest)
if err != nil {
return "", fmt.Errorf("error parsing layer digest %s: %w", entry.Digest, err)
}
layerDigestStr := ellipsize(layerDigest.Encoded(), digestWidth, "")
layerRow := make([]string, 8) //nolint:gomnd
layerRow[colImageNameIndex] = ""
layerRow[colTagIndex] = ""
layerRow[colDigestIndex] = ""
layerRow[colPlatformIndex] = ""
layerRow[colSizeIndex] = size
layerRow[colConfigIndex] = ""
layerRow[colLayersIndex] = layerDigestStr
table.Append(layerRow)
}
layerDigestStr := ellipsize(layerDigest.Encoded(), digestWidth, "")
layerRow := make([]string, 7) //nolint:gomnd
layerRow[colImageNameIndex] = ""
layerRow[colTagIndex] = ""
layerRow[colDigestIndex] = ""
layerRow[colSizeIndex] = size
layerRow[colConfigIndex] = ""
layerRow[colLayersIndex] = layerDigestStr
table.Append(layerRow)
}
}
@ -1011,6 +1176,25 @@ func (img imageStruct) stringPlainText(maxImgNameLen, maxTagLen int) (string, er
return builder.String(), nil
}
func getPlatformStr(platf platform) string {
if platf.Arch == "" && platf.Os == "" {
return "N/A"
}
platform := platf.Os
if platf.Arch != "" {
platform = platform + "/" + platf.Arch
platform = strings.Trim(platform, "/")
if platf.Variant != "" {
platform = platform + "/" + platf.Variant
}
}
return platform
}
func (img imageStruct) stringJSON() (string, error) {
json := jsoniter.ConfigCompatibleWithStandardLibrary
@ -1035,25 +1219,6 @@ type catalogResponse struct {
Repositories []string `json:"repositories"`
}
//nolint:tagliatelle
type manifestResponse struct {
Layers []struct {
MediaType string `json:"mediaType"`
Digest string `json:"digest"`
Size uint64 `json:"size"`
} `json:"layers"`
Annotations struct {
WsTychoStackerStackerYaml string `json:"ws.tycho.stacker.stacker_yaml"`
WsTychoStackerGitVersion string `json:"ws.tycho.stacker.git_version"`
} `json:"annotations"`
Config struct {
Size int `json:"size"`
Digest string `json:"digest"`
MediaType string `json:"mediaType"`
} `json:"config"`
SchemaVersion int `json:"schemaVersion"`
}
func combineServerAndEndpointURL(serverURL, endPoint string) (string, error) {
if !isURL(serverURL) {
return "", zotErrors.ErrInvalidURL
@ -1157,9 +1322,10 @@ func (service searchService) getRepos(ctx context.Context, config searchConfig,
}
const (
imageNameWidth = 32
tagWidth = 24
imageNameWidth = 10
tagWidth = 8
digestWidth = 8
platformWidth = 14
sizeWidth = 8
isSignedWidth = 8
configWidth = 8
@ -1170,9 +1336,10 @@ const (
colTagIndex = 1
colDigestIndex = 2
colConfigIndex = 3
colIsSignedIndex = 4
colLayersIndex = 5
colSizeIndex = 6
colPlatformIndex = 4
colIsSignedIndex = 5
colLayersIndex = 6
colSizeIndex = 7
cveIDWidth = 16
cveSeverityWidth = 8

64
pkg/compliance/v1_0_0/check.go
View file

@ -490,10 +490,10 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) {
repoName := "repo7"
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "test:1.0",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "test:1.0",
}, baseURL, repoName)
So(err, ShouldBeNil)
@ -504,10 +504,10 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) {
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "test:1.0.1",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "test:1.0.1",
}, baseURL, repoName)
So(err, ShouldBeNil)
@ -522,10 +522,10 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) {
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "test:2.0",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "test:2.0",
}, baseURL, repoName)
So(err, ShouldBeNil)
@ -601,10 +601,10 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) {
repoName := "page0"
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: fmt.Sprintf("test:%d.0", index),
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: fmt.Sprintf("test:%d.0", index),
}, baseURL, repoName)
So(err, ShouldBeNil)
@ -742,10 +742,10 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) {
// subpath firsttest
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "test:1.0",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "test:1.0",
}, baseURL, "firsttest/first")
So(err, ShouldBeNil)
@ -757,10 +757,10 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) {
// subpath secondtest
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "test:1.0",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "test:1.0",
}, baseURL, "secondtest/second")
So(err, ShouldBeNil)
@ -776,10 +776,10 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) {
// subpath firsttest
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "test:2.0",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "test:2.0",
}, baseURL, "firsttest/first")
So(err, ShouldBeNil)
@ -791,10 +791,10 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) {
// subpath secondtest
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: "test:2.0",
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: "test:2.0",
}, baseURL, "secondtest/second")
So(err, ShouldBeNil)

8
pkg/extensions/extension_ui_test.go
View file

@ -66,10 +66,10 @@ func TestUIExtension(t *testing.T) {
// Upload a test image
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: tagName,
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: tagName,
}, baseURL, repoName)
So(err, ShouldBeNil)

5
pkg/extensions/lint/lint_test.go
View file

@ -418,6 +418,11 @@ func TestVerifyMandatoryAnnotations(t *testing.T) {
test.CopyTestFiles("../../../test/data", dir)
files, err := os.ReadDir(dir)
So(err, ShouldBeNil)
t.Log("Files in dir:", dir, ": ", files)
ctlr.Config.Storage.RootDirectory = dir
cm := test.NewControllerManager(ctlr)

44
pkg/extensions/search/common/common.go
View file

@ -24,10 +24,15 @@ const (
LabelAnnotationSource = "org.label-schema.vcs-url"
)
type TagInfo struct {
Name string
type Descriptor struct {
Digest godigest.Digest
Timestamp time.Time
MediaType string
}
type TagInfo struct {
Name string
Descriptor Descriptor
Timestamp time.Time
}
func GetRootDir(image string, storeController storage.StoreController) string {
@ -78,6 +83,33 @@ func GetImageDirAndTag(imageName string) (string, string) {
return imageDir, imageTag
}
func GetImageDirAndDigest(imageName string) (string, string) {
var imageDir string
var imageDigest string
if strings.Contains(imageName, "@") {
imageDir, imageDigest, _ = strings.Cut(imageName, "@")
} else {
imageDir = imageName
}
return imageDir, imageDigest
}
// GetImageDirAndReference returns the repo, digest and isTag.
func GetImageDirAndReference(imageName string) (string, string, bool) {
if strings.Contains(imageName, "@") {
repo, digest := GetImageDirAndDigest(imageName)
return repo, digest, false
}
repo, tag := GetImageDirAndTag(imageName)
return repo, tag, true
}
// GetImageLastUpdated This method will return last updated timestamp.
// The Created timestamp is used, but if it is missing, look at the
// history field and, if provided, return the timestamp of last entry in history.
@ -277,3 +309,9 @@ func GetAnnotations(annotations, labels map[string]string) ImageAnnotations {
Authors: authors,
}
}
func ReferenceIsDigest(reference string) bool {
_, err := godigest.Parse(reference)
return err == nil
}

2083
pkg/extensions/search/common/common_test.go
View file

File diff suppressed because it is too large Load diff

32
pkg/extensions/search/common/model.go
View file

@ -13,7 +13,7 @@ type RepoSummary struct {
Name string `json:"name"`
LastUpdated time.Time `json:"lastUpdated"`
Size string `json:"size"`
Platforms []OsArch `json:"platforms"`
Platforms []Platform `json:"platforms"`
Vendors []string `json:"vendors"`
Score int `json:"score"`
NewestImage ImageSummary `json:"newestImage"`
@ -22,28 +22,36 @@ type RepoSummary struct {
type ImageSummary struct {
RepoName string `json:"repoName"`
Tag string `json:"tag"`
Digest string `json:"digest"`
ConfigDigest string `json:"configDigest"`
LastUpdated time.Time `json:"lastUpdated"`
IsSigned bool `json:"isSigned"`
Manifests []ManifestSummary `json:"manifests"`
Size string `json:"size"`
Platform OsArch `json:"platform"`
Vendor string `json:"vendor"`
Score int `json:"score"`
DownloadCount int `json:"downloadCount"`
LastUpdated time.Time `json:"lastUpdated"`
Description string `json:"description"`
IsSigned bool `json:"isSigned"`
Licenses string `json:"licenses"`
Labels string `json:"labels"`
Title string `json:"title"`
Score int `json:"score"`
Source string `json:"source"`
Documentation string `json:"documentation"`
History []LayerHistory `json:"history"`
Layers []LayerSummary `json:"layers"`
Vulnerabilities ImageVulnerabilitySummary `json:"vulnerabilities"`
Authors string `json:"authors"`
Vendor string `json:"vendor"`
Vulnerabilities ImageVulnerabilitySummary `json:"vulnerabilities"`
}
type OsArch struct {
type ManifestSummary struct {
Digest string `json:"digest"`
ConfigDigest string `json:"configDigest"`
LastUpdated time.Time `json:"lastUpdated"`
Size string `json:"size"`
Platform Platform `json:"platform"`
DownloadCount int `json:"downloadCount"`
Layers []LayerSummary `json:"layers"`
History []LayerHistory `json:"history"`
Vulnerabilities ImageVulnerabilitySummary `json:"vulnerabilities"`
}
type Platform struct {
Os string `json:"os"`
Arch string `json:"arch"`
}

72
pkg/extensions/search/common/oci_layout.go
View file

@ -206,7 +206,16 @@ func (olu BaseOciLayoutUtils) GetImageTagsWithTimestamp(repo string) ([]TagInfo,
timeStamp := GetImageLastUpdated(imageInfo)
tagsInfo = append(tagsInfo, TagInfo{Name: val, Timestamp: timeStamp, Digest: digest})
tagsInfo = append(tagsInfo,
TagInfo{
Name: val,
Timestamp: timeStamp,
Descriptor: Descriptor{
Digest: digest,
MediaType: manifest.MediaType,
},
},
)
}
}
@ -327,9 +336,8 @@ func (olu BaseOciLayoutUtils) GetRepoLastUpdated(repo string) (TagInfo, error) {
return latestTag, nil
}
func (olu BaseOciLayoutUtils) GetExpandedRepoInfo(name string) (RepoInfo, error) {
func (olu BaseOciLayoutUtils) GetExpandedRepoInfo(repoName string) (RepoInfo, error) {
repo := RepoInfo{}
repoBlob2Size := make(map[string]int64, 10)
// made up of all manifests, configs and image layers
@ -337,22 +345,22 @@ func (olu BaseOciLayoutUtils) GetExpandedRepoInfo(name string) (RepoInfo, error)
imageSummaries := make([]ImageSummary, 0)
manifestList, err := olu.GetImageManifests(name)
manifestList, err := olu.GetImageManifests(repoName)
if err != nil {
olu.Log.Error().Err(err).Msg("error getting image manifests")
return RepoInfo{}, err
}
lastUpdatedTag, err := olu.GetRepoLastUpdated(name)
lastUpdatedTag, err := olu.GetRepoLastUpdated(repoName)
if err != nil {
olu.Log.Error().Err(err).Msgf("can't get last updated manifest for repo: %s", name)
olu.Log.Error().Err(err).Msgf("can't get last updated manifest for repo: %s", repoName)
return RepoInfo{}, err
}
repoVendorsSet := make(map[string]bool, len(manifestList))
repoPlatformsSet := make(map[string]OsArch, len(manifestList))
repoPlatformsSet := make(map[string]Platform, len(manifestList))
var lastUpdatedImageSummary ImageSummary
@ -367,38 +375,38 @@ func (olu BaseOciLayoutUtils) GetExpandedRepoInfo(name string) (RepoInfo, error)
continue
}
manifest, err := olu.GetImageBlobManifest(name, man.Digest)
manifest, err := olu.GetImageBlobManifest(repoName, man.Digest)
if err != nil {
olu.Log.Error().Err(err).Msg("error getting image manifest blob")
return RepoInfo{}, err
}
isSigned := olu.CheckManifestSignature(name, man.Digest)
isSigned := olu.CheckManifestSignature(repoName, man.Digest)
manifestSize := olu.GetImageManifestSize(name, man.Digest)
manifestSize := olu.GetImageManifestSize(repoName, man.Digest)
olu.Log.Debug().Msg(fmt.Sprintf("%v", man.Digest.String()))
configSize := manifest.Config.Size
repoBlob2Size[man.Digest.String()] = manifestSize
repoBlob2Size[manifest.Config.Digest.String()] = configSize
imageConfigInfo, err := olu.GetImageConfigInfo(name, man.Digest)
imageConfigInfo, err := olu.GetImageConfigInfo(repoName, man.Digest)
if err != nil {
olu.Log.Error().Err(err).Msgf("can't retrieve config info for the image %s %s", name, man.Digest)
olu.Log.Error().Err(err).Msgf("can't retrieve config info for the image %s %s", repoName, man.Digest)
continue
}
opSys, arch := olu.GetImagePlatform(imageConfigInfo)
osArch := OsArch{
platform := Platform{
Os: opSys,
Arch: arch,
}
if opSys != "" || arch != "" {
osArchString := strings.TrimSpace(fmt.Sprintf("%s %s", opSys, arch))
repoPlatformsSet[osArchString] = osArch
platformString := strings.TrimSpace(fmt.Sprintf("%s %s", opSys, arch))
repoPlatformsSet[platformString] = platform
}
layers := make([]LayerSummary, 0)
@ -457,7 +465,7 @@ func (olu BaseOciLayoutUtils) GetExpandedRepoInfo(name string) (RepoInfo, error)
if layersIterator+1 > len(layers) {
olu.Log.Error().Err(errors.ErrBadLayerCount).
Msgf("error on creating layer history for imaeg %s %s", name, man.Digest)
Msgf("error on creating layer history for image %s %s", repoName, man.Digest)
break
}
@ -477,29 +485,35 @@ func (olu BaseOciLayoutUtils) GetExpandedRepoInfo(name string) (RepoInfo, error)
score := 0
imageSummary := ImageSummary{
RepoName: name,
Tag: tag,
RepoName: repoName,
Tag: tag,
Manifests: []ManifestSummary{
{
Digest: manifestDigest,
ConfigDigest: configDigest,
LastUpdated: lastUpdated,
Size: size,
Platform: platform,
Layers: layers,
History: allHistory,
},
},
LastUpdated: lastUpdated,
Digest: manifestDigest,
ConfigDigest: configDigest,
IsSigned: isSigned,
Size: size,
Platform: osArch,
Vendor: annotations.Vendor,
Score: score,
Description: annotations.Description,
Title: annotations.Title,
Documentation: annotations.Documentation,
Licenses: annotations.Licenses,
Labels: annotations.Labels,
Vendor: annotations.Vendor,
Source: annotations.Source,
Layers: layers,
History: allHistory,
}
imageSummaries = append(imageSummaries, imageSummary)
if man.Digest.String() == lastUpdatedTag.Digest.String() {
if man.Digest.String() == lastUpdatedTag.Descriptor.Digest.String() {
lastUpdatedImageSummary = imageSummary
}
}
@ -512,10 +526,10 @@ func (olu BaseOciLayoutUtils) GetExpandedRepoInfo(name string) (RepoInfo, error)
size := strconv.FormatInt(repoSize, 10)
repoPlatforms := make([]OsArch, 0, len(repoPlatformsSet))
repoPlatforms := make([]Platform, 0, len(repoPlatformsSet))
for _, osArch := range repoPlatformsSet {
repoPlatforms = append(repoPlatforms, osArch)
for _, platform := range repoPlatformsSet {
repoPlatforms = append(repoPlatforms, platform)
}
repoVendors := make([]string, 0, len(repoVendorsSet))
@ -526,7 +540,7 @@ func (olu BaseOciLayoutUtils) GetExpandedRepoInfo(name string) (RepoInfo, error)
}
summary := RepoSummary{
Name: name,
Name: repoName,
LastUpdated: lastUpdatedTag.Timestamp,
Size: size,
Platforms: repoPlatforms,

182
pkg/extensions/search/convert/convert_test.go
View file

@ -19,6 +19,7 @@ import (
"zotregistry.io/zot/pkg/extensions/search/common"
"zotregistry.io/zot/pkg/extensions/search/convert"
cveinfo "zotregistry.io/zot/pkg/extensions/search/cve"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
bolt "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper"
. "zotregistry.io/zot/pkg/test"
@ -28,7 +29,7 @@ import (
var ErrTestError = errors.New("TestError")
func TestConvertErrors(t *testing.T) {
Convey("", t, func() {
Convey("Convert Errors", t, func() {
repoDB, err := bolt.NewBoltDBWrapper(bolt.DBParameters{
RootDir: t.TempDir(),
})
@ -59,7 +60,7 @@ func TestConvertErrors(t *testing.T) {
err = repoDB.SetRepoTag("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
repoMetas, manifestMetaMap, _, err := repoDB.SearchRepos(context.Background(), "", repodb.Filter{},
repoMetas, manifestMetaMap, _, _, err := repoDB.SearchRepos(context.Background(), "", repodb.Filter{},
repodb.PageInput{})
So(err, ShouldBeNil)
@ -70,9 +71,11 @@ func TestConvertErrors(t *testing.T) {
ctx,
repoMetas[0],
manifestMetaMap,
map[string]repodb.IndexData{},
convert.SkipQGLField{},
mocks.CveInfoMock{
GetCVESummaryForImageFn: func(image string) (cveinfo.ImageCVESummary, error) {
GetCVESummaryForImageFn: func(repo string, reference string,
) (cveinfo.ImageCVESummary, error) {
return cveinfo.ImageCVESummary{}, ErrTestError
},
},
@ -80,6 +83,167 @@ func TestConvertErrors(t *testing.T) {
So(graphql.GetErrors(ctx).Error(), ShouldContainSubstring, "unable to run vulnerability scan on tag")
})
Convey("ImageIndex2ImageSummary errors", t, func() {
ctx := graphql.WithResponseContext(context.Background(),
graphql.DefaultErrorPresenter, graphql.DefaultRecover)
_, _, err := convert.ImageIndex2ImageSummary(
ctx,
"repo",
"tag",
godigest.FromString("indexDigest"),
true,
repodb.RepoMetadata{},
repodb.IndexData{
IndexBlob: []byte("bad json"),
},
map[string]repodb.ManifestMetadata{},
mocks.CveInfoMock{},
)
So(err, ShouldNotBeNil)
})
Convey("ImageIndex2ImageSummary cve scanning", t, func() {
ctx := graphql.WithResponseContext(context.Background(),
graphql.DefaultErrorPresenter, graphql.DefaultRecover)
_, _, err := convert.ImageIndex2ImageSummary(
ctx,
"repo",
"tag",
godigest.FromString("indexDigest"),
false,
repodb.RepoMetadata{},
repodb.IndexData{
IndexBlob: []byte("{}"),
},
map[string]repodb.ManifestMetadata{},
mocks.CveInfoMock{
GetCVESummaryForImageFn: func(repo, reference string,
) (cveinfo.ImageCVESummary, error) {
return cveinfo.ImageCVESummary{}, ErrTestError
},
},
)
So(err, ShouldBeNil)
})
Convey("ImageManifest2ImageSummary", t, func() {
ctx := graphql.WithResponseContext(context.Background(),
graphql.DefaultErrorPresenter, graphql.DefaultRecover)
_, _, err := convert.ImageManifest2ImageSummary(
ctx,
"repo",
"tag",
godigest.FromString("manifestDigest"),
false,
repodb.RepoMetadata{},
repodb.ManifestMetadata{
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("{}"),
},
mocks.CveInfoMock{
GetCVESummaryForImageFn: func(repo, reference string,
) (cveinfo.ImageCVESummary, error) {
return cveinfo.ImageCVESummary{}, ErrTestError
},
},
)
So(err, ShouldBeNil)
})
Convey("ImageManifest2ManifestSummary", t, func() {
ctx := graphql.WithResponseContext(context.Background(),
graphql.DefaultErrorPresenter, graphql.DefaultRecover)
// with bad config json, error while unmarshaling
_, _, err := convert.ImageManifest2ManifestSummary(
ctx,
"repo",
"tag",
ispec.Descriptor{
Digest: "dig",
MediaType: ispec.MediaTypeImageManifest,
},
false,
repodb.ManifestMetadata{
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("bad json"),
},
mocks.CveInfoMock{
GetCVESummaryForImageFn: func(repo, reference string,
) (cveinfo.ImageCVESummary, error) {
return cveinfo.ImageCVESummary{}, ErrTestError
},
},
)
So(err, ShouldNotBeNil)
// CVE scan using platform
configBlob, err := json.Marshal(ispec.Image{
Platform: ispec.Platform{
OS: "os",
Architecture: "arch",
},
})
So(err, ShouldBeNil)
_, _, err = convert.ImageManifest2ManifestSummary(
ctx,
"repo",
"tag",
ispec.Descriptor{
Digest: "dig",
MediaType: ispec.MediaTypeImageManifest,
},
false,
repodb.ManifestMetadata{
ManifestBlob: []byte("{}"),
ConfigBlob: configBlob,
},
mocks.CveInfoMock{
GetCVESummaryForImageFn: func(repo, reference string,
) (cveinfo.ImageCVESummary, error) {
return cveinfo.ImageCVESummary{}, ErrTestError
},
},
)
So(err, ShouldBeNil)
})
Convey("RepoMeta2ExpandedRepoInfo", t, func() {
ctx := graphql.WithResponseContext(context.Background(),
graphql.DefaultErrorPresenter, graphql.DefaultRecover)
// with bad config json, error while unmarshaling
_, imageSummaries := convert.RepoMeta2ExpandedRepoInfo(
ctx,
repodb.RepoMetadata{
Tags: map[string]repodb.Descriptor{
"tag1": {Digest: "dig", MediaType: ispec.MediaTypeImageManifest},
},
},
map[string]repodb.ManifestMetadata{
"dig": {
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("bad json"),
},
},
map[string]repodb.IndexData{},
convert.SkipQGLField{
Vulnerabilities: false,
},
mocks.CveInfoMock{
GetCVESummaryForImageFn: func(repo, reference string,
) (cveinfo.ImageCVESummary, error) {
return cveinfo.ImageCVESummary{}, ErrTestError
},
}, log.NewLogger("debug", ""),
)
So(len(imageSummaries), ShouldEqual, 0)
})
}
func TestBuildImageInfo(t *testing.T) {
@ -159,7 +323,7 @@ func TestBuildImageInfo(t *testing.T) {
Layers: [][]byte{
layerblob,
},
Tag: "0.0.1",
Reference: "0.0.1",
},
baseURL,
imageName,
@ -174,7 +338,7 @@ func TestBuildImageInfo(t *testing.T) {
imageSummary := convert.BuildImageInfo(imageName, imageName, manifestDigest, ispecManifest,
imageConfig, isSigned)
So(len(imageSummary.Layers), ShouldEqual, len(ispecManifest.Layers))
So(len(imageSummary.Manifests[0].Layers), ShouldEqual, len(ispecManifest.Layers))
imageSummaryLayerSize, err := strconv.Atoi(*imageSummary.Size)
So(err, ShouldBeNil)
So(imageSummaryLayerSize, ShouldEqual, manifestLayersSize)
@ -249,7 +413,7 @@ func TestBuildImageInfo(t *testing.T) {
layerblob,
layerblob2,
},
Tag: "0.0.1",
Reference: "0.0.1",
},
baseURL,
imageName,
@ -264,7 +428,7 @@ func TestBuildImageInfo(t *testing.T) {
imageSummary := convert.BuildImageInfo(imageName, imageName, manifestDigest, ispecManifest,
imageConfig, isSigned)
So(len(imageSummary.Layers), ShouldEqual, len(ispecManifest.Layers))
So(len(imageSummary.Manifests[0].Layers), ShouldEqual, len(ispecManifest.Layers))
imageSummaryLayerSize, err := strconv.Atoi(*imageSummary.Size)
So(err, ShouldBeNil)
So(imageSummaryLayerSize, ShouldEqual, manifestLayersSize)
@ -331,7 +495,7 @@ func TestBuildImageInfo(t *testing.T) {
Layers: [][]byte{
layerblob,
},
Tag: "0.0.1",
Reference: "0.0.1",
},
baseURL,
imageName,
@ -346,7 +510,7 @@ func TestBuildImageInfo(t *testing.T) {
imageSummary := convert.BuildImageInfo(imageName, imageName, manifestDigest, ispecManifest,
imageConfig, isSigned)
So(len(imageSummary.Layers), ShouldEqual, len(ispecManifest.Layers))
So(len(imageSummary.Manifests[0].Layers), ShouldEqual, len(ispecManifest.Layers))
imageSummaryLayerSize, err := strconv.Atoi(*imageSummary.Size)
So(err, ShouldBeNil)
So(imageSummaryLayerSize, ShouldEqual, manifestLayersSize)

115
pkg/extensions/search/convert/oci.go
View file

@ -10,7 +10,6 @@ import (
"zotregistry.io/zot/pkg/extensions/search/common"
"zotregistry.io/zot/pkg/extensions/search/gql_generated"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
)
func BuildImageInfo(repo string, tag string, manifestDigest godigest.Digest,
@ -56,14 +55,23 @@ func BuildImageInfo(repo string, tag string, manifestDigest godigest.Digest,
formattedSize := strconv.FormatInt(size, 10)
imageInfo := &gql_generated.ImageSummary{
RepoName: &repo,
Tag: &tag,
Digest: &formattedManifestDigest,
ConfigDigest: &configDigest,
RepoName: &repo,
Tag: &tag,
Manifests: []*gql_generated.ManifestSummary{
{
Digest: &formattedManifestDigest,
ConfigDigest: &configDigest,
Layers: layers,
Size: &formattedSize,
History: allHistory,
Platform: &gql_generated.Platform{
Os: &imageConfig.OS,
Arch: &imageConfig.Architecture,
},
LastUpdated: &lastUpdated,
},
},
Size: &formattedSize,
Layers: layers,
History: allHistory,
Vendor: &annotations.Vendor,
Description: &annotations.Description,
Title: &annotations.Title,
Documentation: &annotations.Documentation,
@ -71,12 +79,9 @@ func BuildImageInfo(repo string, tag string, manifestDigest godigest.Digest,
Labels: &annotations.Labels,
Source: &annotations.Source,
Authors: &authors,
Vendor: &annotations.Vendor,
LastUpdated: &lastUpdated,
IsSigned: &isSigned,
Platform: &gql_generated.OsArch{
Os: &imageConfig.OS,
Arch: &imageConfig.Architecture,
},
}
return imageInfo
@ -106,15 +111,25 @@ func BuildImageInfo(repo string, tag string, manifestDigest godigest.Digest,
log.Error().Err(zerr.ErrBadLayerCount).Msg("error on creating layer history for ImageSummary")
return &gql_generated.ImageSummary{
RepoName: &repo,
Tag: &tag,
Digest: &formattedManifestDigest,
ConfigDigest: &configDigest,
RepoName: &repo,
Tag: &tag,
Manifests: []*gql_generated.ManifestSummary{
{
Digest: &formattedManifestDigest,
ConfigDigest: &configDigest,
Layers: layers,
Size: &formattedSize,
History: allHistory,
Platform: &gql_generated.Platform{
Os: &imageConfig.OS,
Arch: &imageConfig.Architecture,
},
LastUpdated: &lastUpdated,
},
},
Size: &formattedSize,
Layers: layers,
History: allHistory,
Vendor: &annotations.Vendor,
Description: &annotations.Description,
Vendor: &annotations.Vendor,
Title: &annotations.Title,
Documentation: &annotations.Documentation,
Licenses: &annotations.Licenses,
@ -123,10 +138,6 @@ func BuildImageInfo(repo string, tag string, manifestDigest godigest.Digest,
Authors: &authors,
LastUpdated: &lastUpdated,
IsSigned: &isSigned,
Platform: &gql_generated.OsArch{
Os: &imageConfig.OS,
Arch: &imageConfig.Architecture,
},
}
}
@ -152,27 +163,33 @@ func BuildImageInfo(repo string, tag string, manifestDigest godigest.Digest,
formattedSize := strconv.FormatInt(size, 10)
imageInfo := &gql_generated.ImageSummary{
RepoName: &repo,
Tag: &tag,
Digest: &formattedManifestDigest,
ConfigDigest: &configDigest,
RepoName: &repo,
Tag: &tag,
Manifests: []*gql_generated.ManifestSummary{
{
Digest: &formattedManifestDigest,
ConfigDigest: &configDigest,
Layers: layers,
History: allHistory,
Platform: &gql_generated.Platform{
Os: &imageConfig.OS,
Arch: &imageConfig.Architecture,
},
Size: &formattedSize,
LastUpdated: &lastUpdated,
},
},
Size: &formattedSize,
Layers: layers,
History: allHistory,
Vendor: &annotations.Vendor,
Description: &annotations.Description,
Title: &annotations.Title,
Documentation: &annotations.Documentation,
Licenses: &annotations.Licenses,
Labels: &annotations.Labels,
Source: &annotations.Source,
Vendor: &annotations.Vendor,
Authors: &authors,
LastUpdated: &lastUpdated,
IsSigned: &isSigned,
Platform: &gql_generated.OsArch{
Os: &imageConfig.OS,
Arch: &imageConfig.Architecture,
},
}
return imageInfo
@ -180,23 +197,12 @@ func BuildImageInfo(repo string, tag string, manifestDigest godigest.Digest,
// updateRepoBlobsMap adds all the image blobs and their respective size to the repo blobs map
// and returnes the total size of the image.
func updateRepoBlobsMap(manifestDigest string, manifestSize int64, configDigest string, configSize int64,
layers []ispec.Descriptor, repoBlob2Size map[string]int64,
) int64 {
func updateRepoBlobsMap(imageBlobs map[string]int64, repoBlob2Size map[string]int64) int64 {
imgSize := int64(0)
// add config size
imgSize += configSize
repoBlob2Size[configDigest] = configSize
// add manifest size
imgSize += manifestSize
repoBlob2Size[manifestDigest] = manifestSize
// add layers size
for _, layer := range layers {
repoBlob2Size[layer.Digest.String()] = layer.Size
imgSize += layer.Size
for digest, size := range imageBlobs {
repoBlob2Size[digest] = size
imgSize += size
}
return imgSize
@ -267,14 +273,3 @@ func getAllHistory(manifestContent ispec.Manifest, configContent ispec.Image) (
return allHistory, nil
}
func imageHasSignatures(signatures repodb.ManifestSignatures) bool {
// (sigType, signatures)
for _, sigs := range signatures {
if len(sigs) > 0 {
return true
}
}
return false
}

699
pkg/extensions/search/convert/repodb.go
View file

@ -9,12 +9,15 @@ import (
"time"
"github.com/99designs/gqlgen/graphql"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/vektah/gqlparser/v2/gqlerror"
"zotregistry.io/zot/pkg/extensions/search/common"
cveinfo "zotregistry.io/zot/pkg/extensions/search/cve"
cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model"
"zotregistry.io/zot/pkg/extensions/search/gql_generated"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
)
@ -23,11 +26,12 @@ type SkipQGLField struct {
}
func RepoMeta2RepoSummary(ctx context.Context, repoMeta repodb.RepoMetadata,
manifestMetaMap map[string]repodb.ManifestMetadata, skip SkipQGLField, cveInfo cveinfo.CveInfo,
manifestMetaMap map[string]repodb.ManifestMetadata, indexDataMap map[string]repodb.IndexData,
skip SkipQGLField, cveInfo cveinfo.CveInfo,
) *gql_generated.RepoSummary {
var (
repoLastUpdatedTimestamp = time.Time{}
repoPlatformsSet = map[string]*gql_generated.OsArch{}
repoPlatformsSet = map[string]*gql_generated.Platform{}
repoVendorsSet = map[string]bool{}
lastUpdatedImageSummary *gql_generated.ImageSummary
repoStarCount = repoMeta.Stars
@ -45,107 +49,32 @@ func RepoMeta2RepoSummary(ctx context.Context, repoMeta repodb.RepoMetadata,
)
for tag, descriptor := range repoMeta.Tags {
var (
manifestContent ispec.Manifest
manifestDigest = descriptor.Digest
imageSignatures = repoMeta.Signatures[descriptor.Digest]
)
err := json.Unmarshal(manifestMetaMap[manifestDigest].ManifestBlob, &manifestContent)
imageSummary, imageBlobsMap, err := Descriptor2ImageSummary(ctx, descriptor, repoMeta.Name, tag, true, repoMeta,
manifestMetaMap, indexDataMap, cveInfo)
if err != nil {
graphql.AddError(ctx, gqlerror.Errorf("can't unmarshal manifest blob for image: %s:%s, manifest digest: %s, "+
"error: %s", repoMeta.Name, tag, manifestDigest, err.Error()))
continue
}
var configContent ispec.Image
err = json.Unmarshal(manifestMetaMap[manifestDigest].ConfigBlob, &configContent)
if err != nil {
graphql.AddError(ctx, gqlerror.Errorf("can't unmarshal config blob for image: %s:%s, manifest digest: %s, error: %s",
repoMeta.Name, tag, manifestDigest, err.Error()))
continue
for blobDigest, blobSize := range imageBlobsMap {
repoBlob2Size[blobDigest] = blobSize
}
var (
tag = tag
isSigned = imageHasSignatures(imageSignatures)
configDigest = manifestContent.Config.Digest.String()
configSize = manifestContent.Config.Size
opSys = configContent.OS
arch = configContent.Architecture
osArch = gql_generated.OsArch{Os: &opSys, Arch: &arch}
imageLastUpdated = common.GetImageLastUpdated(configContent)
downloadCount = repoMeta.Statistics[descriptor.Digest].DownloadCount
for _, manifestSummary := range imageSummary.Manifests {
if *manifestSummary.Platform.Os != "" || *manifestSummary.Platform.Arch != "" {
opSys, arch := *manifestSummary.Platform.Os, *manifestSummary.Platform.Arch
size = updateRepoBlobsMap(
manifestDigest, int64(len(manifestMetaMap[manifestDigest].ManifestBlob)),
configDigest, configSize,
manifestContent.Layers,
repoBlob2Size)
imageSize = strconv.FormatInt(size, 10)
)
platformString := strings.TrimSpace(fmt.Sprintf("%s %s", opSys, arch))
repoPlatformsSet[platformString] = &gql_generated.Platform{Os: &opSys, Arch: &arch}
}
annotations := common.GetAnnotations(manifestContent.Annotations, configContent.Config.Labels)
authors := annotations.Authors
if authors == "" {
authors = configContent.Author
repoDownloadCount += manifestMetaMap[*manifestSummary.Digest].DownloadCount
}
historyEntries, err := getAllHistory(manifestContent, configContent)
if err != nil {
graphql.AddError(ctx, gqlerror.Errorf("error generating history on tag %s in repo %s: "+
"manifest digest: %s, error: %s", tag, repoMeta.Name, manifestDigest, err.Error()))
if *imageSummary.Vendor != "" {
repoVendorsSet[*imageSummary.Vendor] = true
}
imageCveSummary := cveinfo.ImageCVESummary{}
imageSummary := gql_generated.ImageSummary{
RepoName: &repoName,
Tag: &tag,
Digest: &manifestDigest,
ConfigDigest: &configDigest,
LastUpdated: &imageLastUpdated,
IsSigned: &isSigned,
Size: &imageSize,
Platform: &osArch,
Vendor: &annotations.Vendor,
DownloadCount: &downloadCount,
Layers: getLayersSummaries(manifestContent),
Description: &annotations.Description,
Title: &annotations.Title,
Documentation: &annotations.Documentation,
Licenses: &annotations.Licenses,
Labels: &annotations.Labels,
Source: &annotations.Source,
Authors: &authors,
History: historyEntries,
Vulnerabilities: &gql_generated.ImageVulnerabilitySummary{
MaxSeverity: &imageCveSummary.MaxSeverity,
Count: &imageCveSummary.Count,
},
}
if annotations.Vendor != "" {
repoVendorsSet[annotations.Vendor] = true
}
if opSys != "" || arch != "" {
osArchString := strings.TrimSpace(fmt.Sprintf("%s %s", opSys, arch))
repoPlatformsSet[osArchString] = &gql_generated.OsArch{Os: &opSys, Arch: &arch}
}
if repoLastUpdatedTimestamp.Equal(time.Time{}) {
// initialize with first time value
repoLastUpdatedTimestamp = imageLastUpdated
lastUpdatedImageSummary = &imageSummary
} else if repoLastUpdatedTimestamp.Before(imageLastUpdated) {
repoLastUpdatedTimestamp = imageLastUpdated
lastUpdatedImageSummary = &imageSummary
}
lastUpdatedImageSummary = UpdateLastUpdatedTimestam(&repoLastUpdatedTimestamp, lastUpdatedImageSummary, imageSummary)
repoDownloadCount += repoMeta.Statistics[descriptor.Digest].DownloadCount
}
@ -158,9 +87,9 @@ func RepoMeta2RepoSummary(ctx context.Context, repoMeta repodb.RepoMetadata,
repoSize := strconv.FormatInt(size, 10)
score := 0
repoPlatforms := make([]*gql_generated.OsArch, 0, len(repoPlatformsSet))
for _, osArch := range repoPlatformsSet {
repoPlatforms = append(repoPlatforms, osArch)
repoPlatforms := make([]*gql_generated.Platform, 0, len(repoPlatformsSet))
for _, platform := range repoPlatformsSet {
repoPlatforms = append(repoPlatforms, platform)
}
repoVendors := make([]*string, 0, len(repoVendorsSet))
@ -173,9 +102,7 @@ func RepoMeta2RepoSummary(ctx context.Context, repoMeta repodb.RepoMetadata,
// We only scan the latest image on the repo for performance reasons
// Check if vulnerability scanning is disabled
if cveInfo != nil && lastUpdatedImageSummary != nil && !skip.Vulnerabilities {
imageName := fmt.Sprintf("%s:%s", repoMeta.Name, *lastUpdatedImageSummary.Tag)
imageCveSummary, err := cveInfo.GetCVESummaryForImage(imageName)
imageCveSummary, err := cveInfo.GetCVESummaryForImage(repoMeta.Name, *lastUpdatedImageSummary.Tag)
if err != nil {
// Log the error, but we should still include the image in results
graphql.AddError(
@ -208,121 +135,393 @@ func RepoMeta2RepoSummary(ctx context.Context, repoMeta repodb.RepoMetadata,
}
}
func UpdateLastUpdatedTimestam(repoLastUpdatedTimestamp *time.Time, lastUpdatedImageSummary *gql_generated.ImageSummary,
imageSummary *gql_generated.ImageSummary,
) *gql_generated.ImageSummary {
newLastUpdatedImageSummary := lastUpdatedImageSummary
if repoLastUpdatedTimestamp.Equal(time.Time{}) {
// initialize with first time value
*repoLastUpdatedTimestamp = *imageSummary.LastUpdated
newLastUpdatedImageSummary = imageSummary
} else if repoLastUpdatedTimestamp.Before(*imageSummary.LastUpdated) {
*repoLastUpdatedTimestamp = *imageSummary.LastUpdated
newLastUpdatedImageSummary = imageSummary
}
return newLastUpdatedImageSummary
}
func Descriptor2ImageSummary(ctx context.Context, descriptor repodb.Descriptor, repo, tag string, skipCVE bool,
repoMeta repodb.RepoMetadata, manifestMetaMap map[string]repodb.ManifestMetadata,
indexDataMap map[string]repodb.IndexData, cveInfo cveinfo.CveInfo,
) (*gql_generated.ImageSummary, map[string]int64, error) {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
return ImageManifest2ImageSummary(ctx, repo, tag, godigest.Digest(descriptor.Digest), skipCVE,
repoMeta, manifestMetaMap[descriptor.Digest], cveInfo)
case ispec.MediaTypeImageIndex:
return ImageIndex2ImageSummary(ctx, repo, tag, godigest.Digest(descriptor.Digest), skipCVE,
repoMeta, indexDataMap[descriptor.Digest], manifestMetaMap, cveInfo)
default:
return &gql_generated.ImageSummary{}, map[string]int64{}, nil
}
}
func ImageIndex2ImageSummary(ctx context.Context, repo, tag string, indexDigest godigest.Digest, skipCVE bool,
repoMeta repodb.RepoMetadata, indexData repodb.IndexData, manifestMetaMap map[string]repodb.ManifestMetadata,
cveInfo cveinfo.CveInfo,
) (*gql_generated.ImageSummary, map[string]int64, error) {
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return &gql_generated.ImageSummary{}, map[string]int64{}, err
}
var (
indexLastUpdated time.Time
isSigned bool
totalIndexSize int64
indexSize string
totalDownloadCount int
maxSeverity string
manifestSummaries = make([]*gql_generated.ManifestSummary, 0, len(indexContent.Manifests))
indexBlobs = make(map[string]int64, 0)
)
for _, descriptor := range indexContent.Manifests {
manifestSummary, manifestBlobs, err := ImageManifest2ManifestSummary(ctx, repo, tag, descriptor, false,
manifestMetaMap[descriptor.Digest.String()], cveInfo)
if err != nil {
return &gql_generated.ImageSummary{}, map[string]int64{}, err
}
manifestSize := int64(0)
for digest, size := range manifestBlobs {
indexBlobs[digest] = size
manifestSize += size
}
if indexLastUpdated.Before(*manifestSummary.LastUpdated) {
indexLastUpdated = *manifestSummary.LastUpdated
}
totalIndexSize += manifestSize
if cvemodel.SeverityValue(*manifestSummary.Vulnerabilities.MaxSeverity) >
cvemodel.SeverityValue(maxSeverity) {
maxSeverity = *manifestSummary.Vulnerabilities.MaxSeverity
}
manifestSummaries = append(manifestSummaries, manifestSummary)
}
for _, signatures := range repoMeta.Signatures[indexDigest.String()] {
if len(signatures) > 0 {
isSigned = true
}
}
imageCveSummary := cveinfo.ImageCVESummary{}
if cveInfo != nil && !skipCVE {
imageCveSummary, err = cveInfo.GetCVESummaryForImage(repo, tag)
if err != nil {
// Log the error, but we should still include the manifest in results
graphql.AddError(ctx, gqlerror.Errorf("unable to run vulnerability scan on tag %s in repo %s: "+
"manifest digest: %s, error: %s", tag, repo, indexDigest, err.Error()))
}
}
indexSize = strconv.FormatInt(totalIndexSize, 10)
annotations := common.GetAnnotations(indexContent.Annotations, map[string]string{})
indexSummary := gql_generated.ImageSummary{
RepoName: &repo,
Tag: &tag,
Manifests: manifestSummaries,
LastUpdated: &indexLastUpdated,
IsSigned: &isSigned,
Size: &indexSize,
DownloadCount: &totalDownloadCount,
Description: &annotations.Description,
Title: &annotations.Title,
Documentation: &annotations.Documentation,
Licenses: &annotations.Licenses,
Labels: &annotations.Labels,
Source: &annotations.Source,
Vendor: &annotations.Vendor,
Vulnerabilities: &gql_generated.ImageVulnerabilitySummary{
MaxSeverity: &imageCveSummary.MaxSeverity,
Count: &imageCveSummary.Count,
},
}
return &indexSummary, indexBlobs, nil
}
func ImageManifest2ImageSummary(ctx context.Context, repo, tag string, digest godigest.Digest, skipCVE bool,
repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata, cveInfo cveinfo.CveInfo,
) (*gql_generated.ImageSummary, map[string]int64, error) {
var (
manifestContent ispec.Manifest
manifestDigest = digest.String()
)
err := json.Unmarshal(manifestMeta.ManifestBlob, &manifestContent)
if err != nil {
graphql.AddError(ctx, gqlerror.Errorf("can't unmarshal manifest blob for image: %s:%s, manifest digest: %s, "+
"error: %s", repo, tag, manifestDigest, err.Error()))
return &gql_generated.ImageSummary{}, map[string]int64{}, err
}
var configContent ispec.Image
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
graphql.AddError(ctx, gqlerror.Errorf("can't unmarshal config blob for image: %s:%s, manifest digest: %s, error: %s",
repo, tag, manifestDigest, err.Error()))
return &gql_generated.ImageSummary{}, map[string]int64{}, err
}
var (
repoName = repo
configDigest = manifestContent.Config.Digest.String()
configSize = manifestContent.Config.Size
imageLastUpdated = common.GetImageLastUpdated(configContent)
downloadCount = repoMeta.Statistics[digest.String()].DownloadCount
isSigned = false
)
opSys := configContent.OS
arch := configContent.Architecture
variant := configContent.Variant
if variant != "" {
arch = arch + "/" + variant
}
platform := gql_generated.Platform{Os: &opSys, Arch: &arch}
for _, signatures := range repoMeta.Signatures[digest.String()] {
if len(signatures) > 0 {
isSigned = true
}
}
size, imageBlobsMap := getImageBlobsInfo(
manifestDigest, int64(len(manifestMeta.ManifestBlob)),
configDigest, configSize,
manifestContent.Layers)
imageSize := strconv.FormatInt(size, 10)
annotations := common.GetAnnotations(manifestContent.Annotations, configContent.Config.Labels)
authors := annotations.Authors
if authors == "" {
authors = configContent.Author
}
historyEntries, err := getAllHistory(manifestContent, configContent)
if err != nil {
graphql.AddError(ctx, gqlerror.Errorf("error generating history on tag %s in repo %s: "+
"manifest digest: %s, error: %s", tag, repo, manifestDigest, err.Error()))
}
imageCveSummary := cveinfo.ImageCVESummary{}
if cveInfo != nil && !skipCVE {
imageCveSummary, err = cveInfo.GetCVESummaryForImage(repo, tag)
if err != nil {
// Log the error, but we should still include the manifest in results
graphql.AddError(ctx, gqlerror.Errorf("unable to run vulnerability scan on tag %s in repo %s: "+
"manifest digest: %s, error: %s", tag, repo, manifestDigest, err.Error()))
}
}
imageSummary := gql_generated.ImageSummary{
RepoName: &repoName,
Tag: &tag,
Manifests: []*gql_generated.ManifestSummary{
{
Digest: &manifestDigest,
ConfigDigest: &configDigest,
LastUpdated: &imageLastUpdated,
Size: &imageSize,
Platform: &platform,
DownloadCount: &downloadCount,
Layers: getLayersSummaries(manifestContent),
History: historyEntries,
Vulnerabilities: &gql_generated.ImageVulnerabilitySummary{
MaxSeverity: &imageCveSummary.MaxSeverity,
Count: &imageCveSummary.Count,
},
},
},
LastUpdated: &imageLastUpdated,
IsSigned: &isSigned,
Size: &imageSize,
DownloadCount: &downloadCount,
Description: &annotations.Description,
Title: &annotations.Title,
Documentation: &annotations.Documentation,
Licenses: &annotations.Licenses,
Labels: &annotations.Labels,
Source: &annotations.Source,
Vendor: &annotations.Vendor,
Authors: &authors,
Vulnerabilities: &gql_generated.ImageVulnerabilitySummary{
MaxSeverity: &imageCveSummary.MaxSeverity,
Count: &imageCveSummary.Count,
},
}
return &imageSummary, imageBlobsMap, nil
}
func ImageManifest2ManifestSummary(ctx context.Context, repo, tag string, descriptor ispec.Descriptor,
skipCVE bool, manifestMeta repodb.ManifestMetadata, cveInfo cveinfo.CveInfo,
) (*gql_generated.ManifestSummary, map[string]int64, error) {
var (
manifestContent ispec.Manifest
digest = descriptor.Digest
)
err := json.Unmarshal(manifestMeta.ManifestBlob, &manifestContent)
if err != nil {
graphql.AddError(ctx, gqlerror.Errorf("can't unmarshal manifest blob for image: %s:%s, manifest digest: %s, "+
"error: %s", repo, tag, digest, err.Error()))
return &gql_generated.ManifestSummary{}, map[string]int64{}, err
}
var configContent ispec.Image
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
graphql.AddError(ctx, gqlerror.Errorf("can't unmarshal config blob for image: %s:%s, manifest digest: %s, error: %s",
repo, tag, digest, err.Error()))
return &gql_generated.ManifestSummary{}, map[string]int64{}, err
}
var (
manifestDigestStr = digest.String()
configDigest = manifestContent.Config.Digest.String()
configSize = manifestContent.Config.Size
imageLastUpdated = common.GetImageLastUpdated(configContent)
downloadCount = manifestMeta.DownloadCount
)
opSys := configContent.OS
arch := configContent.Architecture
variant := configContent.Variant
if variant != "" {
arch = arch + "/" + variant
}
platform := gql_generated.Platform{Os: &opSys, Arch: &arch}
size, imageBlobsMap := getImageBlobsInfo(
manifestDigestStr, int64(len(manifestMeta.ManifestBlob)),
configDigest, configSize,
manifestContent.Layers)
imageSize := strconv.FormatInt(size, 10)
historyEntries, err := getAllHistory(manifestContent, configContent)
if err != nil {
graphql.AddError(ctx, gqlerror.Errorf("error generating history on tag %s in repo %s: "+
"manifest digest: %s, error: %s", tag, repo, manifestDigestStr, err.Error()))
}
imageCveSummary := cveinfo.ImageCVESummary{}
if cveInfo != nil && !skipCVE {
imageCveSummary, err = cveInfo.GetCVESummaryForImage(repo, tag)
if err != nil {
// Log the error, but we should still include the manifest in results
graphql.AddError(ctx, gqlerror.Errorf("unable to run vulnerability scan on tag %s in repo %s: "+
"manifest digest: %s, error: %s", tag, repo, manifestDigestStr, err.Error()))
}
}
manifestSummary := gql_generated.ManifestSummary{
Digest: &manifestDigestStr,
ConfigDigest: &configDigest,
LastUpdated: &imageLastUpdated,
Size: &imageSize,
Platform: &platform,
DownloadCount: &downloadCount,
Layers: getLayersSummaries(manifestContent),
History: historyEntries,
Vulnerabilities: &gql_generated.ImageVulnerabilitySummary{
MaxSeverity: &imageCveSummary.MaxSeverity,
Count: &imageCveSummary.Count,
},
}
return &manifestSummary, imageBlobsMap, nil
}
func getImageBlobsInfo(manifestDigest string, manifestSize int64, configDigest string, configSize int64,
layers []ispec.Descriptor,
) (int64, map[string]int64) {
imageBlobsMap := map[string]int64{}
imageSize := int64(0)
// add config size
imageSize += configSize
imageBlobsMap[configDigest] = configSize
// add manifest size
imageSize += manifestSize
imageBlobsMap[manifestDigest] = manifestSize
// add layers size
for _, layer := range layers {
imageBlobsMap[layer.Digest.String()] = layer.Size
imageSize += layer.Size
}
return imageSize, imageBlobsMap
}
func RepoMeta2ImageSummaries(ctx context.Context, repoMeta repodb.RepoMetadata,
manifestMetaMap map[string]repodb.ManifestMetadata, skip SkipQGLField, cveInfo cveinfo.CveInfo,
manifestMetaMap map[string]repodb.ManifestMetadata, indexDataMap map[string]repodb.IndexData,
skip SkipQGLField, cveInfo cveinfo.CveInfo,
) []*gql_generated.ImageSummary {
imageSummaries := make([]*gql_generated.ImageSummary, 0, len(repoMeta.Tags))
for tag, descriptor := range repoMeta.Tags {
var (
manifestContent ispec.Manifest
manifestDigest = descriptor.Digest
imageSignatures = repoMeta.Signatures[descriptor.Digest]
)
err := json.Unmarshal(manifestMetaMap[manifestDigest].ManifestBlob, &manifestContent)
imageSummary, _, err := Descriptor2ImageSummary(ctx, descriptor, repoMeta.Name, tag, skip.Vulnerabilities,
repoMeta, manifestMetaMap, indexDataMap, cveInfo)
if err != nil {
graphql.AddError(ctx, gqlerror.Errorf("can't unmarshal manifest blob for image: %s:%s, "+
"manifest digest: %s, error: %s", repoMeta.Name, tag, manifestDigest, err.Error()))
continue
}
var configContent ispec.Image
err = json.Unmarshal(manifestMetaMap[manifestDigest].ConfigBlob, &configContent)
if err != nil {
graphql.AddError(ctx, gqlerror.Errorf("can't unmarshal config blob for image: %s:%s, "+
"manifest digest: %s, error: %s", repoMeta.Name, tag, manifestDigest, err.Error()))
continue
}
imageCveSummary := cveinfo.ImageCVESummary{}
// Check if vulnerability scanning is disabled
if cveInfo != nil && !skip.Vulnerabilities {
imageName := fmt.Sprintf("%s:%s", repoMeta.Name, tag)
imageCveSummary, err = cveInfo.GetCVESummaryForImage(imageName)
if err != nil {
// Log the error, but we should still include the manifest in results
graphql.AddError(ctx, gqlerror.Errorf("unable to run vulnerability scan on tag %s in repo %s: "+
"manifest digest: %s, error: %s", tag, repoMeta.Name, manifestDigest, err.Error()))
}
}
imgSize := int64(0)
imgSize += manifestContent.Config.Size
imgSize += int64(len(manifestMetaMap[manifestDigest].ManifestBlob))
for _, layer := range manifestContent.Layers {
imgSize += layer.Size
}
var (
repoName = repoMeta.Name
tag = tag
configDigest = manifestContent.Config.Digest.String()
imageLastUpdated = common.GetImageLastUpdated(configContent)
isSigned = imageHasSignatures(imageSignatures)
imageSize = strconv.FormatInt(imgSize, 10)
os = configContent.OS
arch = configContent.Architecture
osArch = gql_generated.OsArch{Os: &os, Arch: &arch}
downloadCount = repoMeta.Statistics[descriptor.Digest].DownloadCount
)
annotations := common.GetAnnotations(manifestContent.Annotations, configContent.Config.Labels)
authors := annotations.Authors
if authors == "" {
authors = configContent.Author
}
historyEntries, err := getAllHistory(manifestContent, configContent)
if err != nil {
graphql.AddError(ctx, gqlerror.Errorf("error generating history on tag %s in repo %s: "+
"manifest digest: %s, error: %s", tag, repoMeta.Name, manifestDigest, err.Error()))
}
imageSummary := gql_generated.ImageSummary{
RepoName: &repoName,
Tag: &tag,
Digest: &manifestDigest,
ConfigDigest: &configDigest,
LastUpdated: &imageLastUpdated,
IsSigned: &isSigned,
Size: &imageSize,
Platform: &osArch,
Vendor: &annotations.Vendor,
DownloadCount: &downloadCount,
Layers: getLayersSummaries(manifestContent),
Description: &annotations.Description,
Title: &annotations.Title,
Documentation: &annotations.Documentation,
Licenses: &annotations.Licenses,
Labels: &annotations.Labels,
Source: &annotations.Source,
Authors: &authors,
History: historyEntries,
Vulnerabilities: &gql_generated.ImageVulnerabilitySummary{
MaxSeverity: &imageCveSummary.MaxSeverity,
Count: &imageCveSummary.Count,
},
}
imageSummaries = append(imageSummaries, &imageSummary)
imageSummaries = append(imageSummaries, imageSummary)
}
return imageSummaries
}
func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta repodb.RepoMetadata,
manifestMetaMap map[string]repodb.ManifestMetadata, skip SkipQGLField, cveInfo cveinfo.CveInfo,
manifestMetaMap map[string]repodb.ManifestMetadata, indexDataMap map[string]repodb.IndexData,
skip SkipQGLField, cveInfo cveinfo.CveInfo, log log.Logger,
) (*gql_generated.RepoSummary, []*gql_generated.ImageSummary) {
var (
repoLastUpdatedTimestamp = time.Time{}
repoPlatformsSet = map[string]*gql_generated.OsArch{}
repoPlatformsSet = map[string]*gql_generated.Platform{}
repoVendorsSet = map[string]bool{}
lastUpdatedImageSummary *gql_generated.ImageSummary
repoStarCount = repoMeta.Stars
@ -342,104 +541,33 @@ func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta repodb.RepoMetadata
)
for tag, descriptor := range repoMeta.Tags {
var (
manifestContent ispec.Manifest
manifestDigest = descriptor.Digest
imageSignatures = repoMeta.Signatures[descriptor.Digest]
)
err := json.Unmarshal(manifestMetaMap[manifestDigest].ManifestBlob, &manifestContent)
imageSummary, imageBlobs, err := Descriptor2ImageSummary(ctx, descriptor, repoName, tag,
skip.Vulnerabilities, repoMeta, manifestMetaMap, indexDataMap, cveInfo)
if err != nil {
graphql.AddError(ctx, gqlerror.Errorf("can't unmarshal manifest blob for image: %s:%s, manifest digest: %s, "+
"error: %s", repoMeta.Name, tag, manifestDigest, err.Error()))
log.Error().Msgf("repodb: erorr while converting descriptor for image '%s:%s'", repoName, tag)
continue
}
var configContent ispec.Image
for _, manifestSummary := range imageSummary.Manifests {
opSys, arch := *manifestSummary.Platform.Os, *manifestSummary.Platform.Arch
if opSys != "" || arch != "" {
platformString := strings.TrimSpace(fmt.Sprintf("%s %s", opSys, arch))
repoPlatformsSet[platformString] = &gql_generated.Platform{Os: &opSys, Arch: &arch}
}
err = json.Unmarshal(manifestMetaMap[manifestDigest].ConfigBlob, &configContent)
if err != nil {
graphql.AddError(ctx, gqlerror.Errorf("can't unmarshal config blob for image: %s:%s, manifest digest: %s, error: %s",
repoMeta.Name, tag, manifestDigest, err.Error()))
continue
updateRepoBlobsMap(imageBlobs, repoBlob2Size)
}
var (
tag = tag
isSigned = imageHasSignatures(imageSignatures)
configDigest = manifestContent.Config.Digest.String()
configSize = manifestContent.Config.Size
opSys = configContent.OS
arch = configContent.Architecture
osArch = gql_generated.OsArch{Os: &opSys, Arch: &arch}
imageLastUpdated = common.GetImageLastUpdated(configContent)
downloadCount = repoMeta.Statistics[descriptor.Digest].DownloadCount
size = updateRepoBlobsMap(
manifestDigest, int64(len(manifestMetaMap[manifestDigest].ManifestBlob)),
configDigest, configSize,
manifestContent.Layers,
repoBlob2Size)
imageSize = strconv.FormatInt(size, 10)
)
annotations := common.GetAnnotations(manifestContent.Annotations, configContent.Config.Labels)
authors := annotations.Authors
if authors == "" {
authors = configContent.Author
if *imageSummary.Vendor != "" {
repoVendorsSet[*imageSummary.Vendor] = true
}
imageCveSummary := cveinfo.ImageCVESummary{}
lastUpdatedImageSummary = UpdateLastUpdatedTimestam(&repoLastUpdatedTimestamp, lastUpdatedImageSummary, imageSummary)
imageSummary := gql_generated.ImageSummary{
RepoName: &repoName,
Tag: &tag,
Digest: &manifestDigest,
ConfigDigest: &configDigest,
LastUpdated: &imageLastUpdated,
IsSigned: &isSigned,
Size: &imageSize,
Platform: &osArch,
Vendor: &annotations.Vendor,
DownloadCount: &downloadCount,
Layers: getLayersSummaries(manifestContent),
Description: &annotations.Description,
Title: &annotations.Title,
Documentation: &annotations.Documentation,
Licenses: &annotations.Licenses,
Labels: &annotations.Labels,
Source: &annotations.Source,
Authors: &authors,
Vulnerabilities: &gql_generated.ImageVulnerabilitySummary{
MaxSeverity: &imageCveSummary.MaxSeverity,
Count: &imageCveSummary.Count,
},
}
repoDownloadCount += *imageSummary.DownloadCount
imageSummaries = append(imageSummaries, &imageSummary)
if annotations.Vendor != "" {
repoVendorsSet[annotations.Vendor] = true
}
if opSys != "" || arch != "" {
osArchString := strings.TrimSpace(fmt.Sprintf("%s %s", opSys, arch))
repoPlatformsSet[osArchString] = &gql_generated.OsArch{Os: &opSys, Arch: &arch}
}
if repoLastUpdatedTimestamp.Equal(time.Time{}) {
// initialize with first time value
repoLastUpdatedTimestamp = imageLastUpdated
lastUpdatedImageSummary = &imageSummary
} else if repoLastUpdatedTimestamp.Before(imageLastUpdated) {
repoLastUpdatedTimestamp = imageLastUpdated
lastUpdatedImageSummary = &imageSummary
}
repoDownloadCount += repoMeta.Statistics[descriptor.Digest].DownloadCount
imageSummaries = append(imageSummaries, imageSummary)
}
// calculate repo size = sum all manifest, config and layer blobs sizes
@ -450,9 +578,9 @@ func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta repodb.RepoMetadata
repoSize := strconv.FormatInt(size, 10)
score := 0
repoPlatforms := make([]*gql_generated.OsArch, 0, len(repoPlatformsSet))
for _, osArch := range repoPlatformsSet {
repoPlatforms = append(repoPlatforms, osArch)
repoPlatforms := make([]*gql_generated.Platform, 0, len(repoPlatformsSet))
for _, platform := range repoPlatformsSet {
repoPlatforms = append(repoPlatforms, platform)
}
repoVendors := make([]*string, 0, len(repoVendorsSet))
@ -461,13 +589,10 @@ func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta repodb.RepoMetadata
vendor := vendor
repoVendors = append(repoVendors, &vendor)
}
// We only scan the latest image on the repo for performance reasons
// Check if vulnerability scanning is disabled
if cveInfo != nil && lastUpdatedImageSummary != nil && !skip.Vulnerabilities {
imageName := fmt.Sprintf("%s:%s", repoMeta.Name, *lastUpdatedImageSummary.Tag)
imageCveSummary, err := cveInfo.GetCVESummaryForImage(imageName)
imageCveSummary, err := cveInfo.GetCVESummaryForImage(repoMeta.Name, *lastUpdatedImageSummary.Tag)
if err != nil {
// Log the error, but we should still include the image in results
graphql.AddError(

195
pkg/extensions/search/cve/cve.go
View file

@ -7,6 +7,7 @@ import (
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/extensions/search/common"
cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model"
"zotregistry.io/zot/pkg/extensions/search/cve/trivy"
@ -18,15 +19,15 @@ import (
type CveInfo interface {
GetImageListForCVE(repo, cveID string) ([]common.TagInfo, error)
GetImageListWithCVEFixed(repo, cveID string) ([]common.TagInfo, error)
GetCVEListForImage(image string, pageinput PageInput) ([]cvemodel.CVE, PageInfo, error)
GetCVESummaryForImage(image string) (ImageCVESummary, error)
GetCVEListForImage(repo, tag string, pageinput PageInput) ([]cvemodel.CVE, PageInfo, error)
GetCVESummaryForImage(repo, tag string) (ImageCVESummary, error)
CompareSeverities(severity1, severity2 string) int
UpdateDB() error
}
type Scanner interface {
ScanImage(image string) (map[string]cvemodel.CVE, error)
IsImageFormatScannable(image string) (bool, error)
IsImageFormatScannable(repo, tag string) (bool, error)
CompareSeverities(severity1, severity2 string) int
UpdateDB() error
}
@ -66,48 +67,37 @@ func (cveinfo BaseCveInfo) GetImageListForCVE(repo, cveID string) ([]common.TagI
}
for tag, descriptor := range repoMeta.Tags {
manifestDigestStr := descriptor.Digest
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigestStr := descriptor.Digest
manifestDigest, err := godigest.Parse(manifestDigestStr)
if err != nil {
cveinfo.Log.Error().Err(err).Str("repo", repo).Str("tag", tag).
Str("cve-id", cveID).Str("digest", manifestDigestStr).Msg("unable to parse digest")
manifestDigest := godigest.Digest(manifestDigestStr)
return nil, err
}
isScanableImage, err := cveinfo.Scanner.IsImageFormatScannable(repo, tag)
if !isScanableImage || err != nil {
cveinfo.Log.Info().Str("image", repo+":"+tag).Err(err).Msg("image is not scanable")
manifestMeta, err := cveinfo.RepoDB.GetManifestMeta(repo, manifestDigest)
if err != nil {
return nil, err
}
continue
}
var manifestContent ispec.Manifest
cveMap, err := cveinfo.Scanner.ScanImage(getImageString(repo, tag))
if err != nil {
cveinfo.Log.Info().Str("image", repo+":"+tag).Err(err).Msg("image scan failed")
err = json.Unmarshal(manifestMeta.ManifestBlob, &manifestContent)
if err != nil {
cveinfo.Log.Error().Err(err).Str("repo", repo).Str("tag", tag).
Str("cve-id", cveID).Msg("unable to unmashal manifest blob")
continue
}
continue
}
image := fmt.Sprintf("%s:%s", repo, tag)
isValidImage, _ := cveinfo.Scanner.IsImageFormatScannable(image)
if !isValidImage {
continue
}
cveMap, err := cveinfo.Scanner.ScanImage(image)
if err != nil {
continue
}
if _, hasCVE := cveMap[cveID]; hasCVE {
imgList = append(imgList, common.TagInfo{
Name: tag,
Digest: manifestDigest,
})
if _, hasCVE := cveMap[cveID]; hasCVE {
imgList = append(imgList, common.TagInfo{
Name: tag,
Descriptor: common.Descriptor{
Digest: manifestDigest,
MediaType: descriptor.MediaType,
},
})
}
default:
cveinfo.Log.Error().Msgf("type '%s' not supported for scanning", descriptor.MediaType)
}
}
@ -126,67 +116,87 @@ func (cveinfo BaseCveInfo) GetImageListWithCVEFixed(repo, cveID string) ([]commo
vulnerableTags := make([]common.TagInfo, 0)
allTags := make([]common.TagInfo, 0)
var hasCVE bool
for tag, descriptor := range repoMeta.Tags {
manifestDigestStr := descriptor.Digest
manifestDigest, err := godigest.Parse(manifestDigestStr)
if err != nil {
cveinfo.Log.Error().Err(err).Str("repo", repo).Str("tag", tag).
Str("cve-id", cveID).Str("digest", manifestDigestStr).Msg("unable to parse digest")
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest, err := godigest.Parse(manifestDigestStr)
if err != nil {
cveinfo.Log.Error().Err(err).Str("repo", repo).Str("tag", tag).
Str("cve-id", cveID).Str("digest", manifestDigestStr).Msg("unable to parse digest")
continue
}
continue
}
manifestMeta, err := cveinfo.RepoDB.GetManifestMeta(repo, manifestDigest)
if err != nil {
cveinfo.Log.Error().Err(err).Str("repo", repo).Str("tag", tag).
Str("cve-id", cveID).Msg("unable to obtain manifest meta")
manifestMeta, err := cveinfo.RepoDB.GetManifestMeta(repo, manifestDigest)
if err != nil {
cveinfo.Log.Error().Err(err).Str("repo", repo).Str("tag", tag).
Str("cve-id", cveID).Msg("unable to obtain manifest meta")
continue
}
continue
}
var configContent ispec.Image
var configContent ispec.Image
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
cveinfo.Log.Error().Err(err).Str("repo", repo).Str("tag", tag).
Str("cve-id", cveID).Msg("unable to unmashal manifest blob")
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
cveinfo.Log.Error().Err(err).Str("repo", repo).Str("tag", tag).
Str("cve-id", cveID).Msg("unable to unmashal manifest blob")
continue
}
continue
}
tagInfo := common.TagInfo{
Name: tag,
Timestamp: common.GetImageLastUpdated(configContent),
Digest: manifestDigest,
}
tagInfo := common.TagInfo{
Name: tag,
Timestamp: common.GetImageLastUpdated(configContent),
Descriptor: common.Descriptor{Digest: manifestDigest, MediaType: descriptor.MediaType},
}
allTags = append(allTags, tagInfo)
allTags = append(allTags, tagInfo)
image := fmt.Sprintf("%s:%s", repo, tag)
image := fmt.Sprintf("%s:%s", repo, tag)
isValidImage, _ := cveinfo.Scanner.IsImageFormatScannable(image)
if !isValidImage {
cveinfo.Log.Debug().Str("image", image).Str("cve-id", cveID).
Msg("image media type not supported for scanning, adding as a vulnerable image")
isValidImage, err := cveinfo.Scanner.IsImageFormatScannable(repo, tag)
if !isValidImage || err != nil {
cveinfo.Log.Debug().Str("image", image).Str("cve-id", cveID).
Msg("image media type not supported for scanning, adding as a vulnerable image")
vulnerableTags = append(vulnerableTags, tagInfo)
vulnerableTags = append(vulnerableTags, tagInfo)
continue
}
continue
}
cveMap, err := cveinfo.Scanner.ScanImage(image)
if err != nil {
cveinfo.Log.Debug().Str("image", image).Str("cve-id", cveID).
Msg("scanning failed, adding as a vulnerable image")
cveMap, err := cveinfo.Scanner.ScanImage(getImageString(repo, tag))
if err != nil {
cveinfo.Log.Debug().Str("image", image).Str("cve-id", cveID).
Msg("scanning failed, adding as a vulnerable image")
vulnerableTags = append(vulnerableTags, tagInfo)
vulnerableTags = append(vulnerableTags, tagInfo)
continue
}
continue
}
if _, hasCVE := cveMap[cveID]; hasCVE {
vulnerableTags = append(vulnerableTags, tagInfo)
hasCVE = false
for id := range cveMap {
if id == cveID {
hasCVE = true
break
}
}
if hasCVE {
vulnerableTags = append(vulnerableTags, tagInfo)
}
default:
cveinfo.Log.Error().Msgf("media type not supported '%s'", descriptor.MediaType)
return []common.TagInfo{},
fmt.Errorf("media type '%s' is not supported: %w", descriptor.MediaType, errors.ErrNotImplemented)
}
}
@ -205,16 +215,18 @@ func (cveinfo BaseCveInfo) GetImageListWithCVEFixed(repo, cveID string) ([]commo
return fixedTags, nil
}
func (cveinfo BaseCveInfo) GetCVEListForImage(image string, pageInput PageInput) (
func (cveinfo BaseCveInfo) GetCVEListForImage(repo, tag string, pageInput PageInput) (
[]cvemodel.CVE,
PageInfo,
error,
) {
isValidImage, err := cveinfo.Scanner.IsImageFormatScannable(image)
isValidImage, err := cveinfo.Scanner.IsImageFormatScannable(repo, tag)
if !isValidImage {
return []cvemodel.CVE{}, PageInfo{}, err
}
image := getImageString(repo, tag)
cveMap, err := cveinfo.Scanner.ScanImage(image)
if err != nil {
return []cvemodel.CVE{}, PageInfo{}, err
@ -234,7 +246,8 @@ func (cveinfo BaseCveInfo) GetCVEListForImage(image string, pageInput PageInput)
return cveList, pageInfo, nil
}
func (cveinfo BaseCveInfo) GetCVESummaryForImage(image string) (ImageCVESummary, error) {
func (cveinfo BaseCveInfo) GetCVESummaryForImage(repo, tag string,
) (ImageCVESummary, error) {
// There are several cases, expected returned values below:
// not scannable / error during scan - max severity "" - cve count 0 - Errors
// scannable no issues found - max severity "NONE" - cve count 0 - no Errors
@ -244,11 +257,13 @@ func (cveinfo BaseCveInfo) GetCVESummaryForImage(image string) (ImageCVESummary,
MaxSeverity: "",
}
isValidImage, err := cveinfo.Scanner.IsImageFormatScannable(image)
isValidImage, err := cveinfo.Scanner.IsImageFormatScannable(repo, tag)
if !isValidImage {
return imageCVESummary, err
}
image := getImageString(repo, tag)
cveMap, err := cveinfo.Scanner.ScanImage(image)
if err != nil {
return imageCVESummary, err
@ -272,6 +287,16 @@ func (cveinfo BaseCveInfo) GetCVESummaryForImage(image string) (ImageCVESummary,
return imageCVESummary, nil
}
func getImageString(repo, reference string) string {
image := repo + ":" + reference
if common.ReferenceIsDigest(reference) {
image = repo + "@" + reference
}
return image
}
func (cveinfo BaseCveInfo) UpdateDB() error {
return cveinfo.Scanner.UpdateDB()
}

199
pkg/extensions/search/cve/cve_test.go
View file

@ -27,7 +27,6 @@ import (
"zotregistry.io/zot/pkg/api/constants"
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/extensions/search/common"
cveinfo "zotregistry.io/zot/pkg/extensions/search/cve"
cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model"
"zotregistry.io/zot/pkg/log"
@ -148,7 +147,7 @@ func generateTestData(dbDir string) error { //nolint: gocyclo
return err
}
content = fmt.Sprintf(`{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:2a9b097b4e4c613dd8185eba55163201a221909f3d430f8df87cd3639afc5929","size":1240,"annotations":{"org.opencontainers.image.ref.name":"commit-aaa7c6e7-squashfs"},"platform":{"architecture":"amd64","os":"linux"}}]}
content = fmt.Sprint(`{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:2a9b097b4e4c613dd8185eba55163201a221909f3d430f8df87cd3639afc5929","size":1240,"annotations":{"org.opencontainers.image.ref.name":"commit-aaa7c6e7-squashfs"},"platform":{"architecture":"amd64","os":"linux"}}]}
`)
err = makeTestFile(path.Join(dbDir, "zot-squashfs-invalid-blob", "index.json"), content)
@ -156,7 +155,7 @@ func generateTestData(dbDir string) error { //nolint: gocyclo
return err
}
content = fmt.Sprintf(`{"schemaVersion":2,"config"{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:4b37d4133908ac9a3032ba996020f2ad41354a616b071ca7e726a1df18a0f354","size":1691},"layers":[{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:a01a66356aace53222e92fb6fd990b23eb44ab0e58dff6f853fa9f771ecf3ac5","size":54996992},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:91c26d6934ef2b5c5c4d8458af9bfc4ca46cf90c22380193154964abc8298a7a","size":52330496},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:f281a550ca49746cfc6b8f1ac52f8086b3d5845db2ca18fde980dab62ae3bf7d","size":23343104},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:7ee02568717acdda336c9d56d4dc6ea7f3b1c553e43bb0c0ecc6fd3bbd059d1a","size":5910528},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:8fb33b130588b239235dedd560cdf49d29bbf6f2db5419ac68e4592a85c1f416","size":123269120},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:1b49f0b33d4a696bb94d84c9acab3623e2c195bfb446d446a583a2f9f27b04c3","size":113901568}],"annotations":{"com.cisco.stacker.git_version":"7-dev19-63-gaaa7c6e7","ws.tycho.stacker.git_version":"0.3.26"}}
content = fmt.Sprint(`{"schemaVersion":2,"config"{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:4b37d4133908ac9a3032ba996020f2ad41354a616b071ca7e726a1df18a0f354","size":1691},"layers":[{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:a01a66356aace53222e92fb6fd990b23eb44ab0e58dff6f853fa9f771ecf3ac5","size":54996992},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:91c26d6934ef2b5c5c4d8458af9bfc4ca46cf90c22380193154964abc8298a7a","size":52330496},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:f281a550ca49746cfc6b8f1ac52f8086b3d5845db2ca18fde980dab62ae3bf7d","size":23343104},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:7ee02568717acdda336c9d56d4dc6ea7f3b1c553e43bb0c0ecc6fd3bbd059d1a","size":5910528},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:8fb33b130588b239235dedd560cdf49d29bbf6f2db5419ac68e4592a85c1f416","size":123269120},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:1b49f0b33d4a696bb94d84c9acab3623e2c195bfb446d446a583a2f9f27b04c3","size":113901568}],"annotations":{"com.cisco.stacker.git_version":"7-dev19-63-gaaa7c6e7","ws.tycho.stacker.git_version":"0.3.26"}}
`)
err = makeTestFile(path.Join(dbDir, "zot-squashfs-invalid-blob", "blobs/sha256", "2a9b097b4e4c613dd8185eba55163201a221909f3d430f8df87cd3639afc5929"), content)
@ -187,49 +186,49 @@ func generateTestData(dbDir string) error { //nolint: gocyclo
return err
}
content = fmt.Sprintf(`{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:eca04f027f414362596f2632746d8a178362170b9ac9af772011fedcc3877ebb","size":886,"annotations":{"org.opencontainers.image.ref.name":"0.3.25"},"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:45df53588e59759a12bd3eca553cdc9063939baac9a28d7ebb6101e4ec230b76","size":873,"annotations":{"org.opencontainers.image.ref.name":"0.3.22-squashfs"},"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:71448405a4b89539fcfa581afb4dc7d257f98857686b8138b08a1c539f313099","size":886,"annotations":{"org.opencontainers.image.ref.name":"0.3.19"},"platform":{"architecture":"amd64","os":"linux"}}]}`)
content = fmt.Sprint(`{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:eca04f027f414362596f2632746d8a178362170b9ac9af772011fedcc3877ebb","size":886,"annotations":{"org.opencontainers.image.ref.name":"0.3.25"},"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:45df53588e59759a12bd3eca553cdc9063939baac9a28d7ebb6101e4ec230b76","size":873,"annotations":{"org.opencontainers.image.ref.name":"0.3.22-squashfs"},"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:71448405a4b89539fcfa581afb4dc7d257f98857686b8138b08a1c539f313099","size":886,"annotations":{"org.opencontainers.image.ref.name":"0.3.19"},"platform":{"architecture":"amd64","os":"linux"}}]}`)
err = makeTestFile(path.Join(dbDir, "zot-squashfs-test", "index.json"), content)
if err != nil {
return err
}
content = fmt.Sprintf(`{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:c5c2fd2b07ad4cb025cf20936d6bce6085584b8377780599be4da8a91739f0e8","size":1738},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:3414b5ef0ad2f0390daaf55b63c422eeedef6191d47036a69d8ee396fabdce72","size":58993484},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:a3b04fff744c13dfa4883e01fa35e01af8daa7f72d9e9b6b7fad1f28843846b6","size":55631733},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:754f517f58f302190aa94e025c25890c18e1e811127aed1ef25c189278ec4ab0","size":113612795},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:ec004cd43488b803d6e232599e83a3164394d44fcd9f44755fed7b5791087ede","size":108889651}],"annotations":{"ws.tycho.stacker.git_version":"0.3.19"}}`)
content = fmt.Sprint(`{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:c5c2fd2b07ad4cb025cf20936d6bce6085584b8377780599be4da8a91739f0e8","size":1738},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:3414b5ef0ad2f0390daaf55b63c422eeedef6191d47036a69d8ee396fabdce72","size":58993484},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:a3b04fff744c13dfa4883e01fa35e01af8daa7f72d9e9b6b7fad1f28843846b6","size":55631733},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:754f517f58f302190aa94e025c25890c18e1e811127aed1ef25c189278ec4ab0","size":113612795},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:ec004cd43488b803d6e232599e83a3164394d44fcd9f44755fed7b5791087ede","size":108889651}],"annotations":{"ws.tycho.stacker.git_version":"0.3.19"}}`)
err = makeTestFile(path.Join(dbDir, "zot-squashfs-test", "blobs/sha256", "71448405a4b89539fcfa581afb4dc7d257f98857686b8138b08a1c539f313099"), content)
if err != nil {
return err
}
content = fmt.Sprintf(`{"created": "2020-04-08T05:32:49.805795564Z","author": "","architecture": "amd64","os": "linux","config": {"Env": ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"]},"rootfs": {"type": "layers","diff_ids": []},"history": [{"created": "2020-04-08T05:08:43.590117872Z","created_by": "stacker umoci repack"}, {"created": "2020-04-08T05:08:53.213437118Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:12:15.999154739Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:12:31.0513552Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:20:38.068800557Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:21:01.956154957Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:32:24.582132274Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:32:49.805795564Z","created_by": "stacker build","author": "","empty_layer": true}]}`)
content = fmt.Sprint(`{"created": "2020-04-08T05:32:49.805795564Z","author": "","architecture": "amd64","os": "linux","config": {"Env": ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"]},"rootfs": {"type": "layers","diff_ids": []},"history": [{"created": "2020-04-08T05:08:43.590117872Z","created_by": "stacker umoci repack"}, {"created": "2020-04-08T05:08:53.213437118Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:12:15.999154739Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:12:31.0513552Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:20:38.068800557Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:21:01.956154957Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:32:24.582132274Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:32:49.805795564Z","created_by": "stacker build","author": "","empty_layer": true}]}`)
err = makeTestFile(path.Join(dbDir, "zot-squashfs-test", "blobs/sha256", "c5c2fd2b07ad4cb025cf20936d6bce6085584b8377780599be4da8a91739f0e8"), content)
if err != nil {
return err
}
content = fmt.Sprintf(`{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:5f00b5570a5561a6f9b7e66e4f26e2e30c4d09b43a8d3f993f3c1c99be6137a6","size":1740},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:f8b7e41ce10d9a0f614f068326c43431c2777e6fc346f729c2a643bfab24af83","size":59451113},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:9ca9274f196b56a708a7a672d3de88184c0158a30744d355dd0411f3a6850fa6","size":55685756},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:6c1ca50788f93937e9ce9341b564f86cbbcd28e367ed6a57cfc776aee4a9d050","size":113726186},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:d1a92139df86bdf00c818db75bf1ecc860857d142b426e9971a62f5f90e2aa57","size":108755643}],"annotations":{"ws.tycho.stacker.git_version":"0.3.25"}}`)
content = fmt.Sprint(`{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:5f00b5570a5561a6f9b7e66e4f26e2e30c4d09b43a8d3f993f3c1c99be6137a6","size":1740},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:f8b7e41ce10d9a0f614f068326c43431c2777e6fc346f729c2a643bfab24af83","size":59451113},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:9ca9274f196b56a708a7a672d3de88184c0158a30744d355dd0411f3a6850fa6","size":55685756},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:6c1ca50788f93937e9ce9341b564f86cbbcd28e367ed6a57cfc776aee4a9d050","size":113726186},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:d1a92139df86bdf00c818db75bf1ecc860857d142b426e9971a62f5f90e2aa57","size":108755643}],"annotations":{"ws.tycho.stacker.git_version":"0.3.25"}}`)
err = makeTestFile(path.Join(dbDir, "zot-squashfs-test", "blobs/sha256", "eca04f027f414362596f2632746d8a178362170b9ac9af772011fedcc3877ebb"), content)
if err != nil {
return err
}
content = fmt.Sprintf(`{"created": "2020-04-08T05:32:49.805795564Z","author": "","architecture": "amd64","os": "linux","config": {"Env": ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"]},"rootfs": {"type": "layers","diff_ids": []},"history": [{"created": "2020-05-11T18:17:24.516727354Z","created_by": "stacker umoci repack"}, {"created": "2020-04-08T05:08:53.213437118Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:12:15.999154739Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:12:31.0513552Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:20:38.068800557Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:21:01.956154957Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:32:24.582132274Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:32:49.805795564Z","created_by": "stacker build","author": "","empty_layer": true}]}`)
content = fmt.Sprint(`{"created": "2020-04-08T05:32:49.805795564Z","author": "","architecture": "amd64","os": "linux","config": {"Env": ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"]},"rootfs": {"type": "layers","diff_ids": []},"history": [{"created": "2020-05-11T18:17:24.516727354Z","created_by": "stacker umoci repack"}, {"created": "2020-04-08T05:08:53.213437118Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:12:15.999154739Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:12:31.0513552Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:20:38.068800557Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:21:01.956154957Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:32:24.582132274Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:32:49.805795564Z","created_by": "stacker build","author": "","empty_layer": true}]}`)
err = makeTestFile(path.Join(dbDir, "zot-squashfs-test", "blobs/sha256", "5f00b5570a5561a6f9b7e66e4f26e2e30c4d09b43a8d3f993f3c1c99be6137a6"), content)
if err != nil {
return err
}
content = fmt.Sprintf(`{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:1fc1d045b241b04fea54333d76d4f57eb1961f9a314413f02a956b76e77a99f0","size":1218},"layers":[{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:c40d72b1556293c00a3e4b6c64c46ef4c7ae4d876dc18bad942b7d1903e8e5b7","size":54745420},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:4115890e3e2563e545e03f264bfecb0097e24e02306ae3e7668dea52e00c6cc2","size":52213357},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:91859e13e0cf704d5405199d73a2d1a0718391dbb183a77c4cb85d99e923ff57","size":109479329},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:20aef84d8098d47a0643a2f99ce05f0deed957b3a259fb708c538f23ed97cc82","size":103996238}],"annotations":{"ws.tycho.stacker.git_version":"0.3.25"}}`)
content = fmt.Sprint(`{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:1fc1d045b241b04fea54333d76d4f57eb1961f9a314413f02a956b76e77a99f0","size":1218},"layers":[{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:c40d72b1556293c00a3e4b6c64c46ef4c7ae4d876dc18bad942b7d1903e8e5b7","size":54745420},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:4115890e3e2563e545e03f264bfecb0097e24e02306ae3e7668dea52e00c6cc2","size":52213357},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:91859e13e0cf704d5405199d73a2d1a0718391dbb183a77c4cb85d99e923ff57","size":109479329},{"mediaType":"application/vnd.oci.image.layer.squashfs","digest":"sha256:20aef84d8098d47a0643a2f99ce05f0deed957b3a259fb708c538f23ed97cc82","size":103996238}],"annotations":{"ws.tycho.stacker.git_version":"0.3.25"}}`)
err = makeTestFile(path.Join(dbDir, "zot-squashfs-test", "blobs/sha256", "45df53588e59759a12bd3eca553cdc9063939baac9a28d7ebb6101e4ec230b76"), content)
if err != nil {
return err
}
content = fmt.Sprintf(`{"created": "2020-04-08T05:32:49.805795564Z","author": "","architecture": "amd64","os": "linux","config": {"Env": ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"]},"rootfs": {"type": "layers","diff_ids": []},"history": [{"created": "2020-05-11T18:17:24.516727354Z","created_by": "stacker umoci repack"}, {"created": "2020-04-08T05:08:53.213437118Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:12:15.999154739Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-05-11T19:30:02.467956112Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:20:38.068800557Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:21:01.956154957Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:32:24.582132274Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:32:49.805795564Z","created_by": "stacker build","author": "","empty_layer": true}]}`)
content = fmt.Sprint(`{"created": "2020-04-08T05:32:49.805795564Z","author": "","architecture": "amd64","os": "linux","config": {"Env": ["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"]},"rootfs": {"type": "layers","diff_ids": []},"history": [{"created": "2020-05-11T18:17:24.516727354Z","created_by": "stacker umoci repack"}, {"created": "2020-04-08T05:08:53.213437118Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:12:15.999154739Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-05-11T19:30:02.467956112Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:20:38.068800557Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:21:01.956154957Z","created_by": "stacker build","author": "","empty_layer": true}, {"created": "2020-04-08T05:32:24.582132274Z","created_by": "stacker umoci repack","author": ""}, {"created": "2020-04-08T05:32:49.805795564Z","created_by": "stacker build","author": "","empty_layer": true}]}`)
err = makeTestFile(path.Join(dbDir, "zot-squashfs-test", "blobs/sha256", "1fc1d045b241b04fea54333d76d4f57eb1961f9a314413f02a956b76e77a99f0"), content)
if err != nil {
@ -243,21 +242,21 @@ func generateTestData(dbDir string) error { //nolint: gocyclo
return err
}
content = fmt.Sprintf(`{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:eca04f027f414362596f2632746d8a178362170b9ac9af772011fedcc3877ebb","size":886,"annotations":{"org.opencontainers.image.ref.name":"0.3.25"},"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:45df53588e59759a12bd3eca553cdc9063939baac9a28d7ebb6101e4ec230b76","size":873,"annotations":{"org.opencontainers.image.ref.name":"0.3.22-squashfs"},"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:71448405a4b89539fcfa581afb4dc7d257f98857686b8138b08a1c539f313099","size":886,"annotations":{"org.opencontainers.image.ref.name":"0.3.19"},"platform":{"architecture":"amd64","os":"linux"}}]}`)
content = fmt.Sprint(`{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:eca04f027f414362596f2632746d8a178362170b9ac9af772011fedcc3877ebb","size":886,"annotations":{"org.opencontainers.image.ref.name":"0.3.25"},"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:45df53588e59759a12bd3eca553cdc9063939baac9a28d7ebb6101e4ec230b76","size":873,"annotations":{"org.opencontainers.image.ref.name":"0.3.22-squashfs"},"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:71448405a4b89539fcfa581afb4dc7d257f98857686b8138b08a1c539f313099","size":886,"annotations":{"org.opencontainers.image.ref.name":"0.3.19"},"platform":{"architecture":"amd64","os":"linux"}}]}`)
err = makeTestFile(path.Join(dbDir, "zot-invalid-layer", "index.json"), content)
if err != nil {
return err
}
content = fmt.Sprintf(`{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:5f00b5570a5561a6f9b7e66e4f26e2e30c4d09b43a8d3f993f3c1c99be6137a6","size":1740},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:f8b7e41ce10d9a0f614f068326c43431c2777e6fc346f729c2a643bfab24af83","size":59451113},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:9ca9274f196b56a708a7a672d3de88184c0158a30744d355dd0411f3a6850fa6","size":55685756},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:6c1ca50788f93937e9ce9341b564f86cbbcd28e367ed6a57cfc776aee4a9d050","size":113726186},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:d1a92139df86bdf00c818db75bf1ecc860857d142b426e9971a62f5f90e2aa57","size":108755643}],"annotations":{"ws.tycho.stacker.git_version":"0.3.25"}}`)
content = fmt.Sprint(`{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:5f00b5570a5561a6f9b7e66e4f26e2e30c4d09b43a8d3f993f3c1c99be6137a6","size":1740},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:f8b7e41ce10d9a0f614f068326c43431c2777e6fc346f729c2a643bfab24af83","size":59451113},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:9ca9274f196b56a708a7a672d3de88184c0158a30744d355dd0411f3a6850fa6","size":55685756},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:6c1ca50788f93937e9ce9341b564f86cbbcd28e367ed6a57cfc776aee4a9d050","size":113726186},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:d1a92139df86bdf00c818db75bf1ecc860857d142b426e9971a62f5f90e2aa57","size":108755643}],"annotations":{"ws.tycho.stacker.git_version":"0.3.25"}}`)
err = makeTestFile(path.Join(dbDir, "zot-invalid-layer", "blobs/sha256", "eca04f027f414362596f2632746d8a178362170b9ac9af772011fedcc3877ebb"), content)
if err != nil {
return err
}
content = fmt.Sprintf(`{"created":"2020-05-11T19:12:23.239785708Z","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","architecture":"amd64","os":"linux","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"]},"rootfs":{"type":"layers","diff_ids":["sha256:8817d297aa60796f41f559ba688d29b31830854014091233575d474f3a6e808e","sha256:dd5a09481ae1f5caf8d1dbc87bc7f86a01af030796467ba25851ad69964d226d","sha256:a8bce2aaf5ce6f1a5459b72de64927a1e507a911453789bf60df06752222cacd","sha256:dc0b750a934e8f376af23de6dcab1af282967498844a6510aed2c61277f20c11"]},"history":[{"created":"2020-05-11T18:17:24.516727354Z","created_by":"stacker umoci repack"},{"created":"2020-05-11T18:17:33.111086359Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true},{"created":"2020-05-11T18:18:43.147035914Z","created_by":"stacker umoci repack","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI"},{"created":"2020-05-11T18:19:03.346279546Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true},{"created":"2020-05-11T18:27:01.623678656Z","created_by":"stacker umoci repack","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI"},{"created":"2020-05-11T18:27:23.420280147Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true},{"created":"2020-05-11T19:11:54.886053615Z","created_by":"stacker umoci repack","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI"},{"created":"2020-05-11T19:12:23.239785708Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true}]`)
content = fmt.Sprint(`{"created":"2020-05-11T19:12:23.239785708Z","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","architecture":"amd64","os":"linux","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"]},"rootfs":{"type":"layers","diff_ids":["sha256:8817d297aa60796f41f559ba688d29b31830854014091233575d474f3a6e808e","sha256:dd5a09481ae1f5caf8d1dbc87bc7f86a01af030796467ba25851ad69964d226d","sha256:a8bce2aaf5ce6f1a5459b72de64927a1e507a911453789bf60df06752222cacd","sha256:dc0b750a934e8f376af23de6dcab1af282967498844a6510aed2c61277f20c11"]},"history":[{"created":"2020-05-11T18:17:24.516727354Z","created_by":"stacker umoci repack"},{"created":"2020-05-11T18:17:33.111086359Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true},{"created":"2020-05-11T18:18:43.147035914Z","created_by":"stacker umoci repack","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI"},{"created":"2020-05-11T18:19:03.346279546Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true},{"created":"2020-05-11T18:27:01.623678656Z","created_by":"stacker umoci repack","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI"},{"created":"2020-05-11T18:27:23.420280147Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true},{"created":"2020-05-11T19:11:54.886053615Z","created_by":"stacker umoci repack","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI"},{"created":"2020-05-11T19:12:23.239785708Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true}]`)
err = makeTestFile(path.Join(dbDir, "zot-invalid-layer", "blobs/sha256", "5f00b5570a5561a6f9b7e66e4f26e2e30c4d09b43a8d3f993f3c1c99be6137a6"), content)
if err != nil {
@ -271,21 +270,21 @@ func generateTestData(dbDir string) error { //nolint: gocyclo
return err
}
content = fmt.Sprintf(`{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:eca04f027f414362596f2632746d8a178362170b9ac9af772011fedcc3877ebb","size":886,"annotations":{"org.opencontainers.image.ref.name":"0.3.25"},"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:45df53588e59759a12bd3eca553cdc9063939baac9a28d7ebb6101e4ec230b76","size":873,"annotations":{"org.opencontainers.image.ref.name":"0.3.22-squashfs"},"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:71448405a4b89539fcfa581afb4dc7d257f98857686b8138b08a1c539f313099","size":886,"annotations":{"org.opencontainers.image.ref.name":"0.3.19"},"platform":{"architecture":"amd64","os":"linux"}}]}`)
content = fmt.Sprint(`{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:eca04f027f414362596f2632746d8a178362170b9ac9af772011fedcc3877ebb","size":886,"annotations":{"org.opencontainers.image.ref.name":"0.3.25"},"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:45df53588e59759a12bd3eca553cdc9063939baac9a28d7ebb6101e4ec230b76","size":873,"annotations":{"org.opencontainers.image.ref.name":"0.3.22-squashfs"},"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:71448405a4b89539fcfa581afb4dc7d257f98857686b8138b08a1c539f313099","size":886,"annotations":{"org.opencontainers.image.ref.name":"0.3.19"},"platform":{"architecture":"amd64","os":"linux"}}]}`)
err = makeTestFile(path.Join(dbDir, "zot-no-layer", "index.json"), content)
if err != nil {
return err
}
content = fmt.Sprintf(`{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:5f00b5570a5561a6f9b7e66e4f26e2e30c4d09b43a8d3f993f3c1c99be6137a6","size":1740},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:f8b7e41ce10d9a0f614f068326c43431c2777e6fc346f729c2a643bfab24af83","size":59451113},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:9ca9274f196b56a708a7a672d3de88184c0158a30744d355dd0411f3a6850fa6","size":55685756},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:6c1ca50788f93937e9ce9341b564f86cbbcd28e367ed6a57cfc776aee4a9d050","size":113726186},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:d1a92139df86bdf00c818db75bf1ecc860857d142b426e9971a62f5f90e2aa57","size":108755643}],"annotations":{"ws.tycho.stacker.git_version":"0.3.25"}}`)
content = fmt.Sprint(`{"schemaVersion":2,"config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:5f00b5570a5561a6f9b7e66e4f26e2e30c4d09b43a8d3f993f3c1c99be6137a6","size":1740},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:f8b7e41ce10d9a0f614f068326c43431c2777e6fc346f729c2a643bfab24af83","size":59451113},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:9ca9274f196b56a708a7a672d3de88184c0158a30744d355dd0411f3a6850fa6","size":55685756},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:6c1ca50788f93937e9ce9341b564f86cbbcd28e367ed6a57cfc776aee4a9d050","size":113726186},{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:d1a92139df86bdf00c818db75bf1ecc860857d142b426e9971a62f5f90e2aa57","size":108755643}],"annotations":{"ws.tycho.stacker.git_version":"0.3.25"}}`)
err = makeTestFile(path.Join(dbDir, "zot-no-layer", "blobs/sha256", "eca04f027f414362596f2632746d8a178362170b9ac9af772011fedcc3877ebb"), content)
if err != nil {
return err
}
content = fmt.Sprintf(`{"created":"2020-05-11T19:12:23.239785708Z","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","architecture":"amd64","os":"linux","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"]},"rootfs":{"type":"layers","diff_ids":["sha256:8817d297aa60796f41f559ba688d29b31830854014091233575d474f3a6e808e","sha256:dd5a09481ae1f5caf8d1dbc87bc7f86a01af030796467ba25851ad69964d226d","sha256:a8bce2aaf5ce6f1a5459b72de64927a1e507a911453789bf60df06752222cacd","sha256:dc0b750a934e8f376af23de6dcab1af282967498844a6510aed2c61277f20c11"]},"history":[{"created":"2020-05-11T18:17:24.516727354Z","created_by":"stacker umoci repack"},{"created":"2020-05-11T18:17:33.111086359Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true},{"created":"2020-05-11T18:18:43.147035914Z","created_by":"stacker umoci repack","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI"},{"created":"2020-05-11T18:19:03.346279546Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true},{"created":"2020-05-11T18:27:01.623678656Z","created_by":"stacker umoci repack","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI"},{"created":"2020-05-11T18:27:23.420280147Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true},{"created":"2020-05-11T19:11:54.886053615Z","created_by":"stacker umoci repack","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI"},{"created":"2020-05-11T19:12:23.239785708Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true}]`)
content = fmt.Sprint(`{"created":"2020-05-11T19:12:23.239785708Z","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","architecture":"amd64","os":"linux","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"]},"rootfs":{"type":"layers","diff_ids":["sha256:8817d297aa60796f41f559ba688d29b31830854014091233575d474f3a6e808e","sha256:dd5a09481ae1f5caf8d1dbc87bc7f86a01af030796467ba25851ad69964d226d","sha256:a8bce2aaf5ce6f1a5459b72de64927a1e507a911453789bf60df06752222cacd","sha256:dc0b750a934e8f376af23de6dcab1af282967498844a6510aed2c61277f20c11"]},"history":[{"created":"2020-05-11T18:17:24.516727354Z","created_by":"stacker umoci repack"},{"created":"2020-05-11T18:17:33.111086359Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true},{"created":"2020-05-11T18:18:43.147035914Z","created_by":"stacker umoci repack","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI"},{"created":"2020-05-11T18:19:03.346279546Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true},{"created":"2020-05-11T18:27:01.623678656Z","created_by":"stacker umoci repack","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI"},{"created":"2020-05-11T18:27:23.420280147Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true},{"created":"2020-05-11T19:11:54.886053615Z","created_by":"stacker umoci repack","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI"},{"created":"2020-05-11T19:12:23.239785708Z","created_by":"stacker build","author":"root@jenkinsProduction-Atom-Full-Build-c3-master-159CI","empty_layer":true}]`)
err = makeTestFile(path.Join(dbDir, "zot-no-layer", "blobs/sha256", "5f00b5570a5561a6f9b7e66e4f26e2e30c4d09b43a8d3f993f3c1c99be6137a"), content)
if err != nil {
@ -324,50 +323,73 @@ func TestImageFormat(t *testing.T) {
cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "", log)
isValidImage, err := cveInfo.Scanner.IsImageFormatScannable("zot-test")
isValidImage, err := cveInfo.Scanner.IsImageFormatScannable("zot-test", "")
So(err, ShouldNotBeNil)
So(isValidImage, ShouldEqual, false)
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-test:0.0.1")
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-test", "0.0.1")
So(err, ShouldBeNil)
So(isValidImage, ShouldEqual, true)
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-test:0.0.")
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-test", "0.0.")
So(err, ShouldNotBeNil)
So(isValidImage, ShouldEqual, false)
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-noindex-test")
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-noindex-test", "")
So(err, ShouldNotBeNil)
So(isValidImage, ShouldEqual, false)
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot--tet")
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot--tet", "")
So(err, ShouldNotBeNil)
So(isValidImage, ShouldEqual, false)
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-noindex-test")
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-noindex-test", "")
So(err, ShouldNotBeNil)
So(isValidImage, ShouldEqual, false)
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-squashfs-noblobs")
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-squashfs-noblobs", "")
So(err, ShouldNotBeNil)
So(isValidImage, ShouldEqual, false)
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-squashfs-invalid-index")
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-squashfs-invalid-index", "")
So(err, ShouldNotBeNil)
So(isValidImage, ShouldEqual, false)
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-squashfs-invalid-blob")
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-squashfs-invalid-blob", "")
So(err, ShouldNotBeNil)
So(isValidImage, ShouldEqual, false)
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-squashfs-test:0.3.22-squashfs")
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-squashfs-test:0.3.22-squashfs", "")
So(err, ShouldNotBeNil)
So(isValidImage, ShouldEqual, false)
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-nonreadable-test")
isValidImage, err = cveInfo.Scanner.IsImageFormatScannable("zot-nonreadable-test", "")
So(err, ShouldNotBeNil)
So(isValidImage, ShouldEqual, false)
})
Convey("isIndexScanable", t, func() {
log := log.NewLogger("debug", "")
repoDB := &mocks.RepoDBMock{
GetRepoMetaFn: func(repo string) (repodb.RepoMetadata, error) {
return repodb.RepoMetadata{
Tags: map[string]repodb.Descriptor{
"tag": {MediaType: ispec.MediaTypeImageIndex},
},
}, nil
},
}
storeController := storage.StoreController{
DefaultStore: mocks.MockedImageStore{},
}
cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "", log)
isScanable, err := cveInfo.Scanner.IsImageFormatScannable("repo", "tag")
So(err, ShouldBeNil)
So(isScanable, ShouldBeFalse)
})
}
func TestCVESearchDisabled(t *testing.T) {
@ -1023,6 +1045,43 @@ func TestCVEStruct(t *testing.T) {
err = repoDB.SetRepoTag("repo5", "nonexitent-manifest", digest51, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
// ------ Multiarch image
_, _, manifestContent1, err := GetRandomImageComponents(100)
So(err, ShouldBeNil)
manifestContent1Blob, err := json.Marshal(manifestContent1)
So(err, ShouldBeNil)
diestManifestFromIndex1 := godigest.FromBytes(manifestContent1Blob)
err = repoDB.SetManifestData(diestManifestFromIndex1, repodb.ManifestData{
ManifestBlob: manifestContent1Blob,
ConfigBlob: []byte("{}"),
})
So(err, ShouldBeNil)
_, _, manifestContent2, err := GetRandomImageComponents(100)
So(err, ShouldBeNil)
manifestContent2Blob, err := json.Marshal(manifestContent2)
So(err, ShouldBeNil)
diestManifestFromIndex2 := godigest.FromBytes(manifestContent2Blob)
err = repoDB.SetManifestData(diestManifestFromIndex1, repodb.ManifestData{
ManifestBlob: manifestContent2Blob,
ConfigBlob: []byte("{}"),
})
So(err, ShouldBeNil)
indexBlob, err := GetIndexBlobWithManifests(
[]godigest.Digest{diestManifestFromIndex1, diestManifestFromIndex2},
)
So(err, ShouldBeNil)
indexDigest := godigest.FromBytes(indexBlob)
err = repoDB.SetIndexData(indexDigest, repodb.IndexData{
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("repoIndex", "tagIndex", indexDigest, ispec.MediaTypeImageIndex)
So(err, ShouldBeNil)
// RepoDB loaded with initial data, mock the scanner
severities := map[string]int{
"UNKNOWN": 0,
@ -1100,15 +1159,30 @@ func TestCVEStruct(t *testing.T) {
}, nil
}
if image == "repoIndex:tagIndex" {
return map[string]cvemodel.CVE{
"CVE1": {
ID: "CVE1",
Severity: "MEDIUM",
Title: "Title CVE1",
Description: "Description CVE1",
},
}, nil
}
// By default the image has no vulnerabilities
return map[string]cvemodel.CVE{}, nil
},
CompareSeveritiesFn: func(severity1, severity2 string) int {
return severities[severity2] - severities[severity1]
},
IsImageFormatScannableFn: func(image string) (bool, error) {
IsImageFormatScannableFn: func(repo string, reference string) (bool, error) {
if repo == "repoIndex" {
return true, nil
}
// Almost same logic compared to actual Trivy specific implementation
imageDir, inputTag := common.GetImageDirAndTag(image)
imageDir, inputTag := repo, reference
repoMeta, err := repoDB.GetRepoMeta(imageDir)
if err != nil {
@ -1158,51 +1232,51 @@ func TestCVEStruct(t *testing.T) {
t.Log("Test GetCVESummaryForImage")
// Image is found
cveSummary, err := cveInfo.GetCVESummaryForImage("repo1:0.1.0")
cveSummary, err := cveInfo.GetCVESummaryForImage("repo1", "0.1.0")
So(err, ShouldBeNil)
So(cveSummary.Count, ShouldEqual, 1)
So(cveSummary.MaxSeverity, ShouldEqual, "MEDIUM")
cveSummary, err = cveInfo.GetCVESummaryForImage("repo1:1.0.0")
cveSummary, err = cveInfo.GetCVESummaryForImage("repo1", "1.0.0")
So(err, ShouldBeNil)
So(cveSummary.Count, ShouldEqual, 3)
So(cveSummary.MaxSeverity, ShouldEqual, "HIGH")
cveSummary, err = cveInfo.GetCVESummaryForImage("repo1:1.0.1")
cveSummary, err = cveInfo.GetCVESummaryForImage("repo1", "1.0.1")
So(err, ShouldBeNil)
So(cveSummary.Count, ShouldEqual, 2)
So(cveSummary.MaxSeverity, ShouldEqual, "MEDIUM")
cveSummary, err = cveInfo.GetCVESummaryForImage("repo1:1.1.0")
cveSummary, err = cveInfo.GetCVESummaryForImage("repo1", "1.1.0")
So(err, ShouldBeNil)
So(cveSummary.Count, ShouldEqual, 1)
So(cveSummary.MaxSeverity, ShouldEqual, "LOW")
cveSummary, err = cveInfo.GetCVESummaryForImage("repo6:1.0.0")
cveSummary, err = cveInfo.GetCVESummaryForImage("repo6", "1.0.0")
So(err, ShouldBeNil)
So(cveSummary.Count, ShouldEqual, 0)
So(cveSummary.MaxSeverity, ShouldEqual, "NONE")
// Image is not scannable
cveSummary, err = cveInfo.GetCVESummaryForImage("repo2:1.0.0")
cveSummary, err = cveInfo.GetCVESummaryForImage("repo2", "1.0.0")
So(err, ShouldEqual, zerr.ErrScanNotSupported)
So(cveSummary.Count, ShouldEqual, 0)
So(cveSummary.MaxSeverity, ShouldEqual, "")
// Tag is not found
cveSummary, err = cveInfo.GetCVESummaryForImage("repo3:1.0.0")
cveSummary, err = cveInfo.GetCVESummaryForImage("repo3", "1.0.0")
So(err, ShouldEqual, zerr.ErrTagMetaNotFound)
So(cveSummary.Count, ShouldEqual, 0)
So(cveSummary.MaxSeverity, ShouldEqual, "")
// Manifest is not found
cveSummary, err = cveInfo.GetCVESummaryForImage("repo5:nonexitent-manifest")
cveSummary, err = cveInfo.GetCVESummaryForImage("repo5", "nonexitent-manifest")
So(err, ShouldEqual, zerr.ErrManifestDataNotFound)
So(cveSummary.Count, ShouldEqual, 0)
So(cveSummary.MaxSeverity, ShouldEqual, "")
// Repo is not found
cveSummary, err = cveInfo.GetCVESummaryForImage("repo100:1.0.0")
cveSummary, err = cveInfo.GetCVESummaryForImage("repo100", "1.0.0")
So(err, ShouldEqual, zerr.ErrRepoMetaNotFound)
So(cveSummary.Count, ShouldEqual, 0)
So(cveSummary.MaxSeverity, ShouldEqual, "")
@ -1214,14 +1288,14 @@ func TestCVEStruct(t *testing.T) {
}
// Image is found
cveList, pageInfo, err := cveInfo.GetCVEListForImage("repo1:0.1.0", pageInput)
cveList, pageInfo, err := cveInfo.GetCVEListForImage("repo1", "0.1.0", pageInput)
So(err, ShouldBeNil)
So(len(cveList), ShouldEqual, 1)
So(cveList[0].ID, ShouldEqual, "CVE1")
So(pageInfo.ItemCount, ShouldEqual, 1)
So(pageInfo.TotalCount, ShouldEqual, 1)
cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo1:1.0.0", pageInput)
cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo1", "1.0.0", pageInput)
So(err, ShouldBeNil)
So(len(cveList), ShouldEqual, 3)
So(cveList[0].ID, ShouldEqual, "CVE2")
@ -1230,7 +1304,7 @@ func TestCVEStruct(t *testing.T) {
So(pageInfo.ItemCount, ShouldEqual, 3)
So(pageInfo.TotalCount, ShouldEqual, 3)
cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo1:1.0.1", pageInput)
cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo1", "1.0.1", pageInput)
So(err, ShouldBeNil)
So(len(cveList), ShouldEqual, 2)
So(cveList[0].ID, ShouldEqual, "CVE1")
@ -1238,42 +1312,42 @@ func TestCVEStruct(t *testing.T) {
So(pageInfo.ItemCount, ShouldEqual, 2)
So(pageInfo.TotalCount, ShouldEqual, 2)
cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo1:1.1.0", pageInput)
cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo1", "1.1.0", pageInput)
So(err, ShouldBeNil)
So(len(cveList), ShouldEqual, 1)
So(cveList[0].ID, ShouldEqual, "CVE3")
So(pageInfo.ItemCount, ShouldEqual, 1)
So(pageInfo.TotalCount, ShouldEqual, 1)
cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo6:1.0.0", pageInput)
cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo6", "1.0.0", pageInput)
So(err, ShouldBeNil)
So(len(cveList), ShouldEqual, 0)
So(pageInfo.ItemCount, ShouldEqual, 0)
So(pageInfo.TotalCount, ShouldEqual, 0)
// Image is not scannable
cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo2:1.0.0", pageInput)
cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo2", "1.0.0", pageInput)
So(err, ShouldEqual, zerr.ErrScanNotSupported)
So(len(cveList), ShouldEqual, 0)
So(pageInfo.ItemCount, ShouldEqual, 0)
So(pageInfo.TotalCount, ShouldEqual, 0)
// Tag is not found
cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo3:1.0.0", pageInput)
cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo3", "1.0.0", pageInput)
So(err, ShouldEqual, zerr.ErrTagMetaNotFound)
So(len(cveList), ShouldEqual, 0)
So(pageInfo.ItemCount, ShouldEqual, 0)
So(pageInfo.TotalCount, ShouldEqual, 0)
// Manifest is not found
cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo5:nonexitent-manifest", pageInput)
cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo5", "nonexitent-manifest", pageInput)
So(err, ShouldEqual, zerr.ErrManifestDataNotFound)
So(len(cveList), ShouldEqual, 0)
So(pageInfo.ItemCount, ShouldEqual, 0)
So(pageInfo.TotalCount, ShouldEqual, 0)
// Repo is not found
cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo100:1.0.0", pageInput)
cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo100", "1.0.0", pageInput)
So(err, ShouldEqual, zerr.ErrRepoMetaNotFound)
So(len(cveList), ShouldEqual, 0)
So(pageInfo.ItemCount, ShouldEqual, 0)
@ -1388,12 +1462,12 @@ func TestCVEStruct(t *testing.T) {
cveInfo = cveinfo.BaseCveInfo{Log: log, Scanner: faultyScanner, RepoDB: repoDB}
cveSummary, err = cveInfo.GetCVESummaryForImage("repo1:0.1.0")
cveSummary, err = cveInfo.GetCVESummaryForImage("repo1", "0.1.0")
So(err, ShouldNotBeNil)
So(cveSummary.Count, ShouldEqual, 0)
So(cveSummary.MaxSeverity, ShouldEqual, "")
cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo1:0.1.0", pageInput)
cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo1", "0.1.0", pageInput)
So(err, ShouldNotBeNil)
So(cveList, ShouldBeEmpty)
So(pageInfo.ItemCount, ShouldEqual, 0)
@ -1410,5 +1484,32 @@ func TestCVEStruct(t *testing.T) {
// but do not return an error
So(err, ShouldBeNil)
So(len(tagList), ShouldEqual, 0)
cveInfo = cveinfo.BaseCveInfo{Log: log, Scanner: scanner, RepoDB: repoDB}
tagList, err = cveInfo.GetImageListForCVE("repoIndex", "CVE1")
So(err, ShouldBeNil)
So(len(tagList), ShouldEqual, 0)
cveInfo = cveinfo.BaseCveInfo{Log: log, Scanner: mocks.CveScannerMock{
IsImageFormatScannableFn: func(repo, reference string) (bool, error) {
return false, nil
},
}, RepoDB: repoDB}
_, err = cveInfo.GetImageListForCVE("repoIndex", "CVE1")
So(err, ShouldBeNil)
cveInfo = cveinfo.BaseCveInfo{Log: log, Scanner: mocks.CveScannerMock{
IsImageFormatScannableFn: func(repo, reference string) (bool, error) {
return true, nil
},
ScanImageFn: func(image string) (map[string]cvemodel.CVE, error) {
return nil, zerr.ErrTypeAssertionFailed
},
}, RepoDB: repoDB}
_, err = cveInfo.GetImageListForCVE("repoIndex", "CVE1")
So(err, ShouldBeNil)
})
}

20
pkg/extensions/search/cve/model/models.go
View file

@ -15,3 +15,23 @@ type Package struct {
InstalledVersion string `json:"InstalledVersion"`
FixedVersion string `json:"FixedVersion"`
}
const (
None = iota
Low
Medium
High
Critical
)
func SeverityValue(severity string) int {
sevMap := map[string]int{
"NONE": None,
"LOW": Low,
"MEDIUM": Medium,
"HIGH": High,
"CRITICAL": Critical,
}
return sevMap[severity]
}

27
pkg/extensions/search/cve/pagination_test.go
View file

@ -182,7 +182,7 @@ func TestCVEPagination(t *testing.T) {
Convey("Page", func() {
Convey("defaults", func() {
// By default expect unlimitted results sorted by severity
cves, pageInfo, err := cveInfo.GetCVEListForImage("repo1:0.1.0", cveinfo.PageInput{})
cves, pageInfo, err := cveInfo.GetCVEListForImage("repo1", "0.1.0", cveinfo.PageInput{})
So(err, ShouldBeNil)
So(len(cves), ShouldEqual, 5)
So(pageInfo.ItemCount, ShouldEqual, 5)
@ -193,7 +193,7 @@ func TestCVEPagination(t *testing.T) {
previousSeverity = severityToInt[cve.Severity]
}
cves, pageInfo, err = cveInfo.GetCVEListForImage("repo1:1.0.0", cveinfo.PageInput{})
cves, pageInfo, err = cveInfo.GetCVEListForImage("repo1", "1.0.0", cveinfo.PageInput{})
So(err, ShouldBeNil)
So(len(cves), ShouldEqual, 30)
So(pageInfo.ItemCount, ShouldEqual, 30)
@ -211,7 +211,8 @@ func TestCVEPagination(t *testing.T) {
cveIds = append(cveIds, fmt.Sprintf("CVE%d", i))
}
cves, pageInfo, err := cveInfo.GetCVEListForImage("repo1:0.1.0", cveinfo.PageInput{SortBy: cveinfo.AlphabeticAsc})
cves, pageInfo, err := cveInfo.GetCVEListForImage("repo1", "0.1.0",
cveinfo.PageInput{SortBy: cveinfo.AlphabeticAsc})
So(err, ShouldBeNil)
So(len(cves), ShouldEqual, 5)
So(pageInfo.ItemCount, ShouldEqual, 5)
@ -221,7 +222,7 @@ func TestCVEPagination(t *testing.T) {
}
sort.Strings(cveIds)
cves, pageInfo, err = cveInfo.GetCVEListForImage("repo1:1.0.0", cveinfo.PageInput{SortBy: cveinfo.AlphabeticAsc})
cves, pageInfo, err = cveInfo.GetCVEListForImage("repo1", "1.0.0", cveinfo.PageInput{SortBy: cveinfo.AlphabeticAsc})
So(err, ShouldBeNil)
So(len(cves), ShouldEqual, 30)
So(pageInfo.ItemCount, ShouldEqual, 30)
@ -231,7 +232,7 @@ func TestCVEPagination(t *testing.T) {
}
sort.Sort(sort.Reverse(sort.StringSlice(cveIds)))
cves, pageInfo, err = cveInfo.GetCVEListForImage("repo1:1.0.0", cveinfo.PageInput{SortBy: cveinfo.AlphabeticDsc})
cves, pageInfo, err = cveInfo.GetCVEListForImage("repo1", "1.0.0", cveinfo.PageInput{SortBy: cveinfo.AlphabeticDsc})
So(err, ShouldBeNil)
So(len(cves), ShouldEqual, 30)
So(pageInfo.ItemCount, ShouldEqual, 30)
@ -240,7 +241,7 @@ func TestCVEPagination(t *testing.T) {
So(cve.ID, ShouldEqual, cveIds[i])
}
cves, pageInfo, err = cveInfo.GetCVEListForImage("repo1:1.0.0", cveinfo.PageInput{SortBy: cveinfo.SeverityDsc})
cves, pageInfo, err = cveInfo.GetCVEListForImage("repo1", "1.0.0", cveinfo.PageInput{SortBy: cveinfo.SeverityDsc})
So(err, ShouldBeNil)
So(len(cves), ShouldEqual, 30)
So(pageInfo.ItemCount, ShouldEqual, 30)
@ -258,7 +259,7 @@ func TestCVEPagination(t *testing.T) {
cveIds = append(cveIds, fmt.Sprintf("CVE%d", i))
}
cves, pageInfo, err := cveInfo.GetCVEListForImage("repo1:0.1.0", cveinfo.PageInput{
cves, pageInfo, err := cveInfo.GetCVEListForImage("repo1", "0.1.0", cveinfo.PageInput{
Limit: 3,
Offset: 1,
SortBy: cveinfo.AlphabeticAsc,
@ -271,7 +272,7 @@ func TestCVEPagination(t *testing.T) {
So(cves[1].ID, ShouldEqual, "CVE2")
So(cves[2].ID, ShouldEqual, "CVE3")
cves, pageInfo, err = cveInfo.GetCVEListForImage("repo1:0.1.0", cveinfo.PageInput{
cves, pageInfo, err = cveInfo.GetCVEListForImage("repo1", "0.1.0", cveinfo.PageInput{
Limit: 2,
Offset: 1,
SortBy: cveinfo.AlphabeticDsc,
@ -283,7 +284,7 @@ func TestCVEPagination(t *testing.T) {
So(cves[0].ID, ShouldEqual, "CVE3")
So(cves[1].ID, ShouldEqual, "CVE2")
cves, pageInfo, err = cveInfo.GetCVEListForImage("repo1:0.1.0", cveinfo.PageInput{
cves, pageInfo, err = cveInfo.GetCVEListForImage("repo1", "0.1.0", cveinfo.PageInput{
Limit: 3,
Offset: 1,
SortBy: cveinfo.SeverityDsc,
@ -299,7 +300,7 @@ func TestCVEPagination(t *testing.T) {
}
sort.Strings(cveIds)
cves, pageInfo, err = cveInfo.GetCVEListForImage("repo1:1.0.0", cveinfo.PageInput{
cves, pageInfo, err = cveInfo.GetCVEListForImage("repo1", "1.0.0", cveinfo.PageInput{
Limit: 5,
Offset: 20,
SortBy: cveinfo.AlphabeticAsc,
@ -314,7 +315,7 @@ func TestCVEPagination(t *testing.T) {
})
Convey("limit > len(cves)", func() {
cves, pageInfo, err := cveInfo.GetCVEListForImage("repo1:0.1.0", cveinfo.PageInput{
cves, pageInfo, err := cveInfo.GetCVEListForImage("repo1", "0.1.0", cveinfo.PageInput{
Limit: 6,
Offset: 3,
SortBy: cveinfo.AlphabeticAsc,
@ -326,7 +327,7 @@ func TestCVEPagination(t *testing.T) {
So(cves[0].ID, ShouldEqual, "CVE3")
So(cves[1].ID, ShouldEqual, "CVE4")
cves, pageInfo, err = cveInfo.GetCVEListForImage("repo1:0.1.0", cveinfo.PageInput{
cves, pageInfo, err = cveInfo.GetCVEListForImage("repo1", "0.1.0", cveinfo.PageInput{
Limit: 6,
Offset: 3,
SortBy: cveinfo.AlphabeticDsc,
@ -338,7 +339,7 @@ func TestCVEPagination(t *testing.T) {
So(cves[0].ID, ShouldEqual, "CVE1")
So(cves[1].ID, ShouldEqual, "CVE0")
cves, pageInfo, err = cveInfo.GetCVEListForImage("repo1:0.1.0", cveinfo.PageInput{
cves, pageInfo, err = cveInfo.GetCVEListForImage("repo1", "0.1.0", cveinfo.PageInput{
Limit: 6,
Offset: 3,
SortBy: cveinfo.SeverityDsc,

46
pkg/extensions/search/cve/trivy/scanner.go
View file

@ -14,6 +14,7 @@ import (
regTypes "github.com/google/go-containerregistry/pkg/v1/types"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/extensions/search/common"
@ -173,24 +174,49 @@ func (scanner Scanner) runTrivy(opts flag.Options) (types.Report, error) {
return report, nil
}
func (scanner Scanner) IsImageFormatScannable(image string) (bool, error) {
func (scanner Scanner) IsImageFormatScannable(repo, tag string) (bool, error) {
image := repo + ":" + tag
if scanner.cache.Get(image) != nil {
return true, nil
}
imageDir, inputTag := common.GetImageDirAndTag(image)
repoMeta, err := scanner.repoDB.GetRepoMeta(imageDir)
repoMeta, err := scanner.repoDB.GetRepoMeta(repo)
if err != nil {
return false, err
}
manifestDigestStr, ok := repoMeta.Tags[inputTag]
var ok bool
imageDescriptor, ok := repoMeta.Tags[tag]
if !ok {
return false, zerr.ErrTagMetaNotFound
}
manifestDigest, err := godigest.Parse(manifestDigestStr.Digest)
switch imageDescriptor.MediaType {
case ispec.MediaTypeImageManifest:
ok, err := scanner.isManifestScanable(imageDescriptor)
if err != nil {
return ok, errors.Wrapf(err, "image '%s'", image)
}
return ok, nil
case ispec.MediaTypeImageIndex:
ok, err := scanner.isIndexScanable(imageDescriptor)
if err != nil {
return ok, errors.Wrapf(err, "image '%s'", image)
}
return ok, nil
}
return false, nil
}
func (scanner Scanner) isManifestScanable(descriptor repodb.Descriptor) (bool, error) {
manifestDigestStr := descriptor.Digest
manifestDigest, err := godigest.Parse(manifestDigestStr)
if err != nil {
return false, err
}
@ -204,7 +230,7 @@ func (scanner Scanner) IsImageFormatScannable(image string) (bool, error) {
err = json.Unmarshal(manifestData.ManifestBlob, &manifestContent)
if err != nil {
scanner.log.Error().Err(err).Str("image", image).Msg("unable to unmashal manifest blob")
scanner.log.Error().Err(err).Msg("unable to unmashal manifest blob")
return false, zerr.ErrScanNotSupported
}
@ -214,7 +240,7 @@ func (scanner Scanner) IsImageFormatScannable(image string) (bool, error) {
case ispec.MediaTypeImageLayerGzip, ispec.MediaTypeImageLayer, string(regTypes.DockerLayer):
continue
default:
scanner.log.Debug().Str("image", image).
scanner.log.Debug().
Msgf("image media type %s not supported for scanning", imageLayer.MediaType)
return false, zerr.ErrScanNotSupported
@ -224,6 +250,10 @@ func (scanner Scanner) IsImageFormatScannable(image string) (bool, error) {
return true, nil
}
func (scanner Scanner) isIndexScanable(descriptor repodb.Descriptor) (bool, error) {
return false, nil
}
func (scanner Scanner) ScanImage(image string) (map[string]cvemodel.CVE, error) {
if scanner.cache.Get(image) != nil {
return scanner.cache.Get(image), nil

17
pkg/extensions/search/cve/trivy/scanner_internal_test.go
View file

@ -119,6 +119,7 @@ func TestMultipleStoragePath(t *testing.T) {
// Scanning image in default store
cveMap, err := scanner.ScanImage(img0)
So(err, ShouldBeNil)
So(len(cveMap), ShouldEqual, 0)
@ -200,7 +201,7 @@ func TestTrivyLibraryErrors(t *testing.T) {
So(err, ShouldNotBeNil)
// Scanning image with invalid input to trigger a scanner error
opts = scanner.getTrivyOptions("nonexisting_image:0.0.1")
opts = scanner.getTrivyOptions("nilnonexisting_image:0.0.1")
_, err = scanner.runTrivy(opts)
So(err, ShouldNotBeNil)
@ -358,43 +359,43 @@ func TestImageScannable(t *testing.T) {
scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", log)
Convey("Valid image should be scannable", t, func() {
result, err := scanner.IsImageFormatScannable("repo1:valid")
result, err := scanner.IsImageFormatScannable("repo1", "valid")
So(err, ShouldBeNil)
So(result, ShouldBeTrue)
})
Convey("Image with layers of unsupported types should be unscannable", t, func() {
result, err := scanner.IsImageFormatScannable("repo1:unscannable-layer")
result, err := scanner.IsImageFormatScannable("repo1", "unscannable-layer")
So(err, ShouldNotBeNil)
So(result, ShouldBeFalse)
})
Convey("Image with unmarshable manifests should be unscannable", t, func() {
result, err := scanner.IsImageFormatScannable("repo1:unmarshable")
result, err := scanner.IsImageFormatScannable("repo1", "unmarshable")
So(err, ShouldNotBeNil)
So(result, ShouldBeFalse)
})
Convey("Image with missing manifest meta should be unscannable", t, func() {
result, err := scanner.IsImageFormatScannable("repo1:missing")
result, err := scanner.IsImageFormatScannable("repo1", "missing")
So(err, ShouldNotBeNil)
So(result, ShouldBeFalse)
})
Convey("Image with invalid manifest digest should be unscannable", t, func() {
result, err := scanner.IsImageFormatScannable("repo1:invalid-digest")
result, err := scanner.IsImageFormatScannable("repo1", "invalid-digest")
So(err, ShouldNotBeNil)
So(result, ShouldBeFalse)
})
Convey("Image with unknown tag should be unscannable", t, func() {
result, err := scanner.IsImageFormatScannable("repo1:unknown-tag")
result, err := scanner.IsImageFormatScannable("repo1", "unknown-tag")
So(err, ShouldNotBeNil)
So(result, ShouldBeFalse)
})
Convey("Image with unknown repo should be unscannable", t, func() {
result, err := scanner.IsImageFormatScannable("unknown-repo:sometag")
result, err := scanner.IsImageFormatScannable("unknown-repo", "sometag")
So(err, ShouldNotBeNil)
So(result, ShouldBeFalse)
})

2
pkg/extensions/search/digest/digest.go
View file

@ -55,7 +55,7 @@ func (digestinfo DigestInfo) GetImageTagsByDigest(repo, digest string) ([]ImageI
tags := []*string{}
// Check the image manigest in index.json matches the search digest
// Check the image manifest in index.json matches the search digest
// This is a blob with mediaType application/vnd.oci.image.manifest.v1+json
if strings.Contains(manifest.Digest.String(), digest) {
tags = append(tags, &val)

69
pkg/extensions/search/digest/digest_test.go
View file

@ -65,11 +65,11 @@ func testSetup(t *testing.T) (string, string, *digestinfo.DigestInfo) {
subRootDir := subDir
// Test images used/copied:
// IMAGE NAME TAG DIGEST CONFIG LAYERS SIZE
// zot-test 0.0.1 2bacca16 adf3bb6c 76MB
// 2d473b07 76MB
// zot-cve-test 0.0.1 63a795ca 8dd57e17 75MB
// 7a0437f0 75MB
// IMAGE NAME TAG DIGEST OS/ARCH CONFIG LAYERS SIZE
// zot-test 0.0.1 2bacca16 linux/amd64 adf3bb6c 76MB
// 2d473b07 76MB
// zot-cve-test 0.0.1 63a795ca linux/amd64 8dd57e17 75MB
// 7a0437f0 75MB
err := os.Mkdir(subDir+"/a", 0o700)
if err != nil {
@ -159,9 +159,20 @@ func TestDigestSearchHTTP(t *testing.T) {
So(resp.StatusCode(), ShouldEqual, 422)
// "sha" should match all digests in all images
query := `{
ImageListForDigest(id:"sha") {
Results {
RepoName Tag
Manifests {
Digest ConfigDigest Size
Layers { Digest }
}
Size
}
}
}`
resp, err = resty.R().Get(
baseURL + constants.FullSearchPrefix + `?query={ImageListForDigest(id:"sha")` +
`{Results{RepoName%20Tag%20Digest%20ConfigDigest%20Size%20Layers%20{%20Digest}}}}`,
baseURL + constants.FullSearchPrefix + "?query=" + url.QueryEscape(query),
)
So(resp, ShouldNotBeNil)
So(err, ShouldBeNil)
@ -178,7 +189,7 @@ func TestDigestSearchHTTP(t *testing.T) {
// GetTestBlobDigest("zot-test", "manifest").Encoded() should match the manifest of 1 image
gqlQuery := url.QueryEscape(`{ImageListForDigest(id:"` + GetTestBlobDigest("zot-test", "manifest").Encoded() + `")
{Results{RepoName Tag Digest ConfigDigest Size Layers { Digest }}}}`)
{Results{RepoName Tag Manifests {Digest ConfigDigest Size Layers { Digest }}}}}`)
targetURL := baseURL + constants.FullSearchPrefix + `?query=` + gqlQuery
resp, err = resty.R().Get(targetURL)
@ -196,7 +207,7 @@ func TestDigestSearchHTTP(t *testing.T) {
// GetTestBlobDigest("zot-test", "config").Encoded() should match the config of 1 image.
gqlQuery = url.QueryEscape(`{ImageListForDigest(id:"` + GetTestBlobDigest("zot-test", "config").Encoded() + `")
{Results{RepoName Tag Digest ConfigDigest Size Layers { Digest }}}}`)
{Results{RepoName Tag Manifests {Digest ConfigDigest Size Layers { Digest }}}}}`)
targetURL = baseURL + constants.FullSearchPrefix + `?query=` + gqlQuery
resp, err = resty.R().Get(targetURL)
@ -215,7 +226,7 @@ func TestDigestSearchHTTP(t *testing.T) {
// Call should return {"data":{"ImageListForDigest":[{"Name":"zot-cve-test","Tags":["0.0.1"]}]}}
// GetTestBlobDigest("zot-cve-test", "layer").Encoded() should match the layer of 1 image
gqlQuery = url.QueryEscape(`{ImageListForDigest(id:"` + GetTestBlobDigest("zot-cve-test", "layer").Encoded() + `")
{Results{RepoName Tag Digest ConfigDigest Size Layers { Digest }}}}`)
{Results{RepoName Tag Manifests {Digest ConfigDigest Size Layers { Digest }}}}}`)
targetURL = baseURL + constants.FullSearchPrefix + `?query=` + gqlQuery
resp, err = resty.R().Get(
@ -236,9 +247,20 @@ func TestDigestSearchHTTP(t *testing.T) {
// Call should return {"data":{"ImageListForDigest":[]}}
// "1111111" should match 0 images
query = `
{
ImageListForDigest(id:"1111111") {
Results {
RepoName Tag
Manifests {
Digest ConfigDigest Size
Layers { Digest }
}
}
}
}`
resp, err = resty.R().Get(
baseURL + constants.FullSearchPrefix + `?query={ImageListForDigest(id:"1111111")` +
`{Results{RepoName%20Tag%20Digest%20ConfigDigest%20Size%20Layers%20{%20Digest}}}}`,
baseURL + constants.FullSearchPrefix + "?query=" + url.QueryEscape(query),
)
So(resp, ShouldNotBeNil)
So(err, ShouldBeNil)
@ -250,9 +272,14 @@ func TestDigestSearchHTTP(t *testing.T) {
So(len(responseStruct.ImgListForDigest.Results), ShouldEqual, 0)
// Call should return {"errors": [{....}]", data":null}}
query = `{
ImageListForDigest(id:"1111111") {
Results {
RepoName Tag343s
}
}`
resp, err = resty.R().Get(
baseURL + constants.FullSearchPrefix + `?query={ImageListForDigest(id:"1111111")` +
`{Results{RepoName%20Tag343s}}}`,
baseURL + constants.FullSearchPrefix + "?query=" + url.QueryEscape(query),
)
So(resp, ShouldNotBeNil)
So(err, ShouldBeNil)
@ -306,9 +333,19 @@ func TestDigestSearchHTTPSubPaths(t *testing.T) {
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 422)
query := `{
ImageListForDigest(id:"sha") {
Results {
RepoName Tag
Manifests {
Digest ConfigDigest Size
Layers { Digest }
}
}
}
}`
resp, err = resty.R().Get(
baseURL + constants.FullSearchPrefix + `?query={ImageListForDigest(id:"sha")` +
`{Results{RepoName%20Tag%20Digest%20ConfigDigest%20Size%20Layers%20{%20Digest}}}}`,
baseURL + constants.FullSearchPrefix + "?query=" + url.QueryEscape(query),
)
So(resp, ShouldNotBeNil)
So(err, ShouldBeNil)

1547
pkg/extensions/search/gql_generated/generated.go
View file

File diff suppressed because it is too large Load diff

75
pkg/extensions/search/gql_generated/models_gen.go
View file

@ -91,28 +91,18 @@ type ImageSummary struct {
RepoName *string `json:"RepoName"`
// Tag identifying the image within the repository
Tag *string `json:"Tag"`
// Digest of the manifest file associated with this image
Digest *string `json:"Digest"`
// Digest of the config file associated with this image
ConfigDigest *string `json:"ConfigDigest"`
// Timestamp of the last modification done to the image (from config or the last updated layer)
LastUpdated *time.Time `json:"LastUpdated"`
// True if the image has a signature associated with it, false otherwise
IsSigned *bool `json:"IsSigned"`
// Total size of the files associated with this image (manigest, config, layers)
// List of manifests for all supported versions of the image for different operating systems and architectures
Manifests []*ManifestSummary `json:"Manifests"`
// Total size of the files associated with all images (manifest, config, layers)
Size *string `json:"Size"`
// OS and architecture supported by this image
Platform *OsArch `json:"Platform"`
// Vendor associated with this image, the distributing entity, organization or individual
Vendor *string `json:"Vendor"`
// Integer used to rank search results by relevance
Score *int `json:"Score"`
// Number of downloads of the manifest of this image
DownloadCount *int `json:"DownloadCount"`
// Information on the layers of this image
Layers []*LayerSummary `json:"Layers"`
// Timestamp of the last modification done to the image (from config or the last updated layer)
LastUpdated *time.Time `json:"LastUpdated"`
// Human-readable description of the software packaged in the image
Description *string `json:"Description"`
// True if the image has a signature associated with it, false otherwise
IsSigned *bool `json:"IsSigned"`
// License(s) under which contained software is distributed as an SPDX License Expression
Licenses *string `json:"Licenses"`
// Labels associated with this image
@ -120,16 +110,18 @@ type ImageSummary struct {
Labels *string `json:"Labels"`
// Human-readable title of the image
Title *string `json:"Title"`
// Integer used to rank search results by relevance
Score *int `json:"Score"`
// URL to get source code for building the image
Source *string `json:"Source"`
// URL to get documentation on the image
Documentation *string `json:"Documentation"`
// Information about the history of the specific image, see LayerHistory
History []*LayerHistory `json:"History"`
// Short summary of the identified CVEs
Vulnerabilities *ImageVulnerabilitySummary `json:"Vulnerabilities"`
// Vendor associated with this image, the distributing entity, organization or individual
Vendor *string `json:"Vendor"`
// Contact details of the people or organization responsible for the image
Authors *string `json:"Authors"`
// Short summary of the identified CVEs
Vulnerabilities *ImageVulnerabilitySummary `json:"Vulnerabilities"`
}
// Contains summary of vulnerabilities found in a specific image
@ -158,14 +150,27 @@ type LayerSummary struct {
Score *int `json:"Score"`
}
// Contains details about the OS and architecture of the image
type OsArch struct {
// The name of the operating system which the image is built to run on,
// Should be values listed in the Go Language document https://go.dev/doc/install/source#environment
Os *string `json:"Os"`
// The name of the compilation architecture which the image is built to run on,
// Should be values listed in the Go Language document https://go.dev/doc/install/source#environment
Arch *string `json:"Arch"`
// Details about a specific version of an image for a certain operating system and architecture.
type ManifestSummary struct {
// Digest of the manifest file associated with this image
Digest *string `json:"Digest"`
// Digest of the config file associated with this image
ConfigDigest *string `json:"ConfigDigest"`
// Timestamp of the last update to an image inside this repository
LastUpdated *time.Time `json:"LastUpdated"`
// Total size of the files associated with this manifest (manifest, config, layers)
Size *string `json:"Size"`
// OS and architecture supported by this image
Platform *Platform `json:"Platform"`
// Total numer of image manifest downloads from this repository
DownloadCount *int `json:"DownloadCount"`
// List of layers matching the search criteria
// NOTE: the actual search logic for layers is not implemented at the moment
Layers []*LayerSummary `json:"Layers"`
// Information about the history of the specific image, see LayerHistory
History []*LayerHistory `json:"History"`
// Short summary of the identified CVEs
Vulnerabilities *ImageVulnerabilitySummary `json:"Vulnerabilities"`
}
// Contains the name of the package, the current installed version and the version where the CVE was fixed
@ -215,6 +220,16 @@ type PaginatedReposResult struct {
Results []*RepoSummary `json:"Results"`
}
// Contains details about the OS and architecture of the image
type Platform struct {
// The name of the operating system which the image is built to run on,
// Should be values listed in the Go Language document https://go.dev/doc/install/source#environment
Os *string `json:"Os"`
// The name of the compilation architecture which the image is built to run on,
// Should be values listed in the Go Language document https://go.dev/doc/install/source#environment
Arch *string `json:"Arch"`
}
// A referrer is an object which has a reference to a another object
type Referrer struct {
// Referrer MediaType
@ -248,7 +263,7 @@ type RepoSummary struct {
// Total size of the files within this repository
Size *string `json:"Size"`
// List of platforms supported by this repository
Platforms []*OsArch `json:"Platforms"`
Platforms []*Platform `json:"Platforms"`
// Vendors associated with this image, the distributing entities, organizations or individuals
Vendors []*string `json:"Vendors"`
// Integer used to rank search results by relevance

345
pkg/extensions/search/resolver.go
View file

@ -7,6 +7,7 @@ package search
import (
"context"
"encoding/json"
"fmt"
"sort"
"strings"
@ -137,13 +138,14 @@ func getImageListForDigest(ctx context.Context, digest string, repoDB repodb.Rep
}
// get all repos
reposMeta, manifestMetaMap, pageInfo, err := repoDB.FilterTags(ctx, FilterByDigest(digest), pageInput)
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := repoDB.FilterTags(ctx, FilterByDigest(digest), pageInput)
if err != nil {
return &gql_generated.PaginatedImagesResult{}, err
}
for _, repoMeta := range reposMeta {
imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, skip, cveInfo)
imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap,
skip, cveInfo)
imageList = append(imageList, imageSummaries...)
}
@ -157,7 +159,7 @@ func getImageListForDigest(ctx context.Context, digest string, repoDB repodb.Rep
}, nil
}
func getImageSummary(ctx context.Context, repo, tag string, repoDB repodb.RepoDB,
func getImageSummary(ctx context.Context, repo, tag string, digest *string, repoDB repodb.RepoDB,
cveInfo cveinfo.CveInfo, log log.Logger, //nolint:unparam
) (
*gql_generated.ImageSummary, error,
@ -172,28 +174,115 @@ func getImageSummary(ctx context.Context, repo, tag string, repoDB repodb.RepoDB
return nil, gqlerror.Errorf("can't find image: %s:%s", repo, tag)
}
manifestDigest := manifestDescriptor.Digest
for t := range repoMeta.Tags {
if t != tag {
delete(repoMeta.Tags, t)
}
}
manifestMeta, err := repoDB.GetManifestMeta(repo, godigest.Digest(manifestDigest))
if err != nil {
return nil, err
}
var (
manifestMetaMap = map[string]repodb.ManifestMetadata{}
indexDataMap = map[string]repodb.IndexData{}
)
manifestMetaMap := map[string]repodb.ManifestMetadata{
manifestDigest: manifestMeta,
switch manifestDescriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := manifestDescriptor.Digest
if digest != nil && *digest != manifestDigest {
return nil, fmt.Errorf("resolver: can't get ManifestData for digest %s for image '%s:%s' %w",
manifestDigest, repo, tag, zerr.ErrManifestDataNotFound)
}
manifestData, err := repoDB.GetManifestData(godigest.Digest(manifestDigest))
if err != nil {
return nil, err
}
manifestMetaMap[manifestDigest] = repodb.ManifestMetadata{
ManifestBlob: manifestData.ManifestBlob,
ConfigBlob: manifestData.ConfigBlob,
}
case ispec.MediaTypeImageIndex:
indexDigest := manifestDescriptor.Digest
indexData, err := repoDB.GetIndexData(godigest.Digest(indexDigest))
if err != nil {
return nil, err
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return nil, err
}
if digest != nil {
manifestDigest := *digest
digestFound := false
for _, manifest := range indexContent.Manifests {
if manifest.Digest.String() == manifestDigest {
digestFound = true
break
}
}
if !digestFound {
return nil, fmt.Errorf("resolver: can't get ManifestData for digest %s for image '%s:%s' %w",
manifestDigest, repo, tag, zerr.ErrManifestDataNotFound)
}
manifestData, err := repoDB.GetManifestData(godigest.Digest(manifestDigest))
if err != nil {
return nil, fmt.Errorf("resolver: can't get ManifestData for digest %s for image '%s:%s' %w",
manifestDigest, repo, tag, err)
}
manifestMetaMap[manifestDigest] = repodb.ManifestMetadata{
ManifestBlob: manifestData.ManifestBlob,
ConfigBlob: manifestData.ConfigBlob,
}
// We update the tag descriptor to be the manifest descriptor with digest specified in the
// 'digest' parameter. We treat it as a standalone image.
repoMeta.Tags[tag] = repodb.Descriptor{
Digest: manifestDigest,
MediaType: ispec.MediaTypeImageManifest,
}
break
}
for _, manifest := range indexContent.Manifests {
manifestData, err := repoDB.GetManifestData(manifest.Digest)
if err != nil {
return nil, fmt.Errorf("resolver: can't get ManifestData for digest %s for image '%s:%s' %w",
manifest.Digest, repo, tag, err)
}
manifestMetaMap[manifest.Digest.String()] = repodb.ManifestMetadata{
ManifestBlob: manifestData.ManifestBlob,
ConfigBlob: manifestData.ConfigBlob,
}
}
indexDataMap[indexDigest] = indexData
default:
log.Error().Msgf("resolver: media type '%s' not supported", manifestDescriptor.MediaType)
}
skip := convert.SkipQGLField{
Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Vulnerabilities"),
}
imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo)
imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, skip, cveInfo)
if len(imageSummaries) == 0 {
return &gql_generated.ImageSummary{}, nil
}
return imageSummaries[0], nil
}
@ -217,13 +306,17 @@ func getCVEListForImage(
),
}
_, copyImgTag := common.GetImageDirAndTag(image)
repo, ref, isTag := common.GetImageDirAndReference(image)
if copyImgTag == "" {
if ref == "" {
return &gql_generated.CVEResultForImage{}, gqlerror.Errorf("no reference provided")
}
cveList, pageInfo, err := cveInfo.GetCVEListForImage(image, pageInput)
if !isTag {
return &gql_generated.CVEResultForImage{}, gqlerror.Errorf("reference by digest not supported")
}
cveList, pageInfo, err := cveInfo.GetCVEListForImage(repo, ref, pageInput)
if err != nil {
return &gql_generated.CVEResultForImage{}, err
}
@ -262,7 +355,7 @@ func getCVEListForImage(
}
return &gql_generated.CVEResultForImage{
Tag: &copyImgTag,
Tag: &ref,
CVEList: cveids,
Page: &gql_generated.PageInfo{
TotalCount: pageInfo.TotalCount,
@ -276,7 +369,7 @@ func FilterByTagInfo(tagsInfo []common.TagInfo) repodb.FilterFunc {
manifestDigest := godigest.FromBytes(manifestMeta.ManifestBlob).String()
for _, tagInfo := range tagsInfo {
if tagInfo.Digest.String() == manifestDigest {
if tagInfo.Descriptor.Digest.String() == manifestDigest {
return true
}
}
@ -342,13 +435,14 @@ func getImageListForCVE(
}
// get all repos
reposMeta, manifestMetaMap, pageInfo, err := repoDB.FilterTags(ctx, FilterByTagInfo(affectedImages), pageInput)
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := repoDB.FilterTags(ctx,
FilterByTagInfo(affectedImages), pageInput)
if err != nil {
return &gql_generated.PaginatedImagesResult{}, err
}
for _, repoMeta := range reposMeta {
imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, skip, cveInfo)
imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo)
imageList = append(imageList, imageSummaries...)
}
@ -403,7 +497,7 @@ func getImageListWithCVEFixed(
}
// get all repos
reposMeta, manifestMetaMap, pageInfo, err := repoDB.FilterTags(ctx, FilterByTagInfo(tagsInfo), pageInput)
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := repoDB.FilterTags(ctx, FilterByTagInfo(tagsInfo), pageInput)
if err != nil {
return &gql_generated.PaginatedImagesResult{}, err
}
@ -413,7 +507,7 @@ func getImageListWithCVEFixed(
continue
}
imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, skip, cveInfo)
imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo)
imageList = append(imageList, imageSummaries...)
}
@ -452,13 +546,14 @@ func repoListWithNewestImage(
),
}
reposMeta, manifestMetaMap, pageInfo, err := repoDB.SearchRepos(ctx, "", repodb.Filter{}, pageInput)
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := repoDB.SearchRepos(ctx, "", repodb.Filter{}, pageInput)
if err != nil {
return &gql_generated.PaginatedReposResult{}, err
}
for _, repoMeta := range reposMeta {
repoSummary := convert.RepoMeta2RepoSummary(ctx, repoMeta, manifestMetaMap, skip, cveInfo)
repoSummary := convert.RepoMeta2RepoSummary(ctx, repoMeta, manifestMetaMap, indexDataMap,
skip, cveInfo)
repos = append(repos, repoSummary)
}
@ -507,13 +602,14 @@ func globalSearch(ctx context.Context, query string, repoDB repodb.RepoDB, filte
),
}
reposMeta, manifestMetaMap, pageInfo, err := repoDB.SearchRepos(ctx, query, localFilter, pageInput)
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := repoDB.SearchRepos(ctx, query, localFilter, pageInput)
if err != nil {
return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, []*gql_generated.LayerSummary{}, err
}
for _, repoMeta := range reposMeta {
repoSummary := convert.RepoMeta2RepoSummary(ctx, repoMeta, manifestMetaMap, skip, cveInfo)
repoSummary := convert.RepoMeta2RepoSummary(ctx, repoMeta, manifestMetaMap, indexDataMap,
skip, cveInfo)
repos = append(repos, repoSummary)
}
@ -537,13 +633,13 @@ func globalSearch(ctx context.Context, query string, repoDB repodb.RepoDB, filte
),
}
reposMeta, manifestMetaMap, pageInfo, err := repoDB.SearchTags(ctx, query, localFilter, pageInput)
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := repoDB.SearchTags(ctx, query, localFilter, pageInput)
if err != nil {
return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, []*gql_generated.LayerSummary{}, err
}
for _, repoMeta := range reposMeta {
imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, skip, cveInfo)
imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo)
images = append(images, imageSummaries...)
}
@ -563,7 +659,7 @@ func canSkipField(preloads map[string]bool, s string) bool {
return !fieldIsPresent
}
func derivedImageList(ctx context.Context, image string, repoDB repodb.RepoDB,
func derivedImageList(ctx context.Context, image string, digest *string, repoDB repodb.RepoDB,
requestedPage *gql_generated.PageInput,
cveInfo cveinfo.CveInfo, log log.Logger,
) (*gql_generated.PaginatedImagesResult, error) {
@ -590,7 +686,7 @@ func derivedImageList(ctx context.Context, image string, repoDB repodb.RepoDB,
return &gql_generated.PaginatedImagesResult{}, gqlerror.Errorf("no reference provided")
}
searchedImage, err := getImageSummary(ctx, imageRepo, imageTag, repoDB, cveInfo, log)
searchedImage, err := getImageSummary(ctx, imageRepo, imageTag, digest, repoDB, cveInfo, log)
if err != nil {
if errors.Is(err, zerr.ErrRepoMetaNotFound) {
return &gql_generated.PaginatedImagesResult{}, gqlerror.Errorf("repository: not found")
@ -600,7 +696,7 @@ func derivedImageList(ctx context.Context, image string, repoDB repodb.RepoDB,
}
// we need all available tags
reposMeta, manifestMetaMap, pageInfo, err := repoDB.FilterTags(ctx,
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := repoDB.FilterTags(ctx,
filterDerivedImages(searchedImage),
pageInput)
if err != nil {
@ -608,7 +704,7 @@ func derivedImageList(ctx context.Context, image string, repoDB repodb.RepoDB,
}
for _, repoMeta := range reposMeta {
summary := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, skip, cveInfo)
summary := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo)
derivedList = append(derivedList, summary...)
}
@ -641,37 +737,42 @@ func filterDerivedImages(image *gql_generated.ImageSummary) repodb.FilterFunc {
return false
}
manifestDigest := godigest.FromBytes(manifestMeta.ManifestBlob).String()
if manifestDigest == *image.Digest {
return false
}
for i := range image.Manifests {
manifestDigest := godigest.FromBytes(manifestMeta.ManifestBlob).String()
if manifestDigest == *image.Manifests[i].Digest {
return false
}
imageLayers := image.Manifests[i].Layers
imageLayers := image.Layers
addImageToList = false
layers := imageManifest.Layers
addImageToList = false
layers := imageManifest.Layers
sameLayer := 0
sameLayer := 0
for _, l := range imageLayers {
for _, k := range layers {
if k.Digest.String() == *l.Digest {
sameLayer++
for _, l := range imageLayers {
for _, k := range layers {
if k.Digest.String() == *l.Digest {
sameLayer++
}
}
}
// if all layers are the same
if sameLayer == len(imageLayers) {
// it's a derived image
addImageToList = true
}
if addImageToList {
return true
}
}
// if all layers are the same
if sameLayer == len(imageLayers) {
// it's a derived image
addImageToList = true
}
return addImageToList
return false
}
}
func baseImageList(ctx context.Context, image string, repoDB repodb.RepoDB,
func baseImageList(ctx context.Context, image string, digest *string, repoDB repodb.RepoDB,
requestedPage *gql_generated.PageInput,
cveInfo cveinfo.CveInfo, log log.Logger,
) (*gql_generated.PaginatedImagesResult, error) {
@ -699,7 +800,7 @@ func baseImageList(ctx context.Context, image string, repoDB repodb.RepoDB,
return &gql_generated.PaginatedImagesResult{}, gqlerror.Errorf("no reference provided")
}
searchedImage, err := getImageSummary(ctx, imageRepo, imageTag, repoDB, cveInfo, log)
searchedImage, err := getImageSummary(ctx, imageRepo, imageTag, digest, repoDB, cveInfo, log)
if err != nil {
if errors.Is(err, zerr.ErrRepoMetaNotFound) {
return &gql_generated.PaginatedImagesResult{}, gqlerror.Errorf("repository: not found")
@ -709,7 +810,7 @@ func baseImageList(ctx context.Context, image string, repoDB repodb.RepoDB,
}
// we need all available tags
reposMeta, manifestMetaMap, pageInfo, err := repoDB.FilterTags(ctx,
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := repoDB.FilterTags(ctx,
filterBaseImages(searchedImage),
pageInput)
if err != nil {
@ -717,7 +818,7 @@ func baseImageList(ctx context.Context, image string, repoDB repodb.RepoDB,
}
for _, repoMeta := range reposMeta {
summary := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, skip, cveInfo)
summary := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo)
imageSummaries = append(imageSummaries, summary...)
}
@ -743,42 +844,45 @@ func filterBaseImages(image *gql_generated.ImageSummary) repodb.FilterFunc {
return func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
var addImageToList bool
var imageManifest ispec.Manifest
var manifestContent ispec.Manifest
err := json.Unmarshal(manifestMeta.ManifestBlob, &imageManifest)
err := json.Unmarshal(manifestMeta.ManifestBlob, &manifestContent)
if err != nil {
return false
}
manifestDigest := godigest.FromBytes(manifestMeta.ManifestBlob).String()
if manifestDigest == *image.Digest {
return false
}
for i := range image.Manifests {
manifestDigest := godigest.FromBytes(manifestMeta.ManifestBlob).String()
if manifestDigest == *image.Manifests[i].Digest {
return false
}
imageLayers := image.Layers
addImageToList = true
addImageToList = true
layers := imageManifest.Layers
for _, l := range manifestContent.Layers {
foundLayer := false
for _, l := range layers {
foundLayer := false
for _, k := range image.Manifests[i].Layers {
if l.Digest.String() == *k.Digest {
foundLayer = true
for _, k := range imageLayers {
if l.Digest.String() == *k.Digest {
foundLayer = true
break
}
}
if !foundLayer {
addImageToList = false
break
}
}
if !foundLayer {
addImageToList = false
break
if addImageToList {
return true
}
}
return addImageToList
return false
}
}
@ -921,31 +1025,88 @@ func expandedRepoInfo(ctx context.Context, repo string, repoDB repodb.RepoDB, cv
return &gql_generated.RepoInfo{}, err
}
manifestMetaMap := map[string]repodb.ManifestMetadata{}
var (
manifestMetaMap = map[string]repodb.ManifestMetadata{}
indexDataMap = map[string]repodb.IndexData{}
)
for tag, descriptor := range repoMeta.Tags {
digest := descriptor.Digest
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
digest := descriptor.Digest
if _, alreadyDownloaded := manifestMetaMap[digest]; alreadyDownloaded {
continue
if _, alreadyDownloaded := manifestMetaMap[digest]; alreadyDownloaded {
continue
}
manifestMeta, err := repoDB.GetManifestMeta(repo, godigest.Digest(digest))
if err != nil {
graphql.AddError(ctx, errors.Wrapf(err,
"resolver: failed to get manifest meta for image %s:%s with manifest digest %s", repo, tag, digest))
continue
}
manifestMetaMap[digest] = manifestMeta
case ispec.MediaTypeImageIndex:
digest := descriptor.Digest
if _, alreadyDownloaded := indexDataMap[digest]; alreadyDownloaded {
continue
}
indexData, err := repoDB.GetIndexData(godigest.Digest(digest))
if err != nil {
graphql.AddError(ctx, errors.Wrapf(err,
"resolver: failed to get manifest meta for image %s:%s with manifest digest %s", repo, tag, digest))
continue
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
graphql.AddError(ctx, errors.Wrapf(err,
"resolver: failed to unmarshal index content for image %s:%s with digest %s", repo, tag, digest))
continue
}
var errorOccured bool
for _, descriptor := range indexContent.Manifests {
manifestMeta, err := repoDB.GetManifestMeta(repo, descriptor.Digest)
if err != nil {
graphql.AddError(ctx, errors.Wrapf(err,
"resolver: failed to get manifest meta with digest '%s' for multiarch image %s:%s",
digest, repo, tag),
)
errorOccured = true
break
}
manifestMetaMap[descriptor.Digest.String()] = manifestMeta
}
if errorOccured {
continue
}
indexDataMap[digest] = indexData
default:
}
manifestMeta, err := repoDB.GetManifestMeta(repo, godigest.Digest(digest))
if err != nil {
graphql.AddError(ctx, errors.Wrapf(err,
"resolver: failed to get manifest meta for image %s:%s with manifest digest %s", repo, tag, digest))
continue
}
manifestMetaMap[digest] = manifestMeta
}
skip := convert.SkipQGLField{
Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Summary.NewestImage.Vulnerabilities"),
Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Summary.NewestImage.Vulnerabilities") &&
canSkipField(convert.GetPreloads(ctx), "Images.Vulnerabilities"),
}
repoSummary, imageSummaries := convert.RepoMeta2ExpandedRepoInfo(ctx, repoMeta, manifestMetaMap, skip, cveInfo)
repoSummary, imageSummaries := convert.RepoMeta2ExpandedRepoInfo(ctx, repoMeta, manifestMetaMap, indexDataMap,
skip, cveInfo, log)
dateSortedImages := make(timeSlice, 0, len(imageSummaries))
for _, imgSummary := range imageSummaries {
@ -1005,7 +1166,7 @@ func getImageList(ctx context.Context, repo string, repoDB repodb.RepoDB, cveInf
}
// reposMeta, manifestMetaMap, err := repoDB.SearchRepos(ctx, repo, repodb.Filter{}, pageInput)
reposMeta, manifestMetaMap, pageInfo, err := repoDB.FilterTags(ctx,
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := repoDB.FilterTags(ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
return true
},
@ -1018,7 +1179,7 @@ func getImageList(ctx context.Context, repo string, repoDB repodb.RepoDB, cveInf
if repoMeta.Name != repo && repo != "" {
continue
}
imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, skip, cveInfo)
imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo)
imageList = append(imageList, imageSummaries...)
}

631
pkg/extensions/search/resolver_test.go
View file

File diff suppressed because it is too large Load diff

102
pkg/extensions/search/schema.graphql
View file

@ -124,53 +124,33 @@ type ImageSummary {
"""
Tag: String
"""
Digest of the manifest file associated with this image
List of manifests for all supported versions of the image for different operating systems and architectures
"""
Digest: String
Manifests: [ManifestSummary]
"""
Digest of the config file associated with this image
"""
ConfigDigest: String
"""
Timestamp of the last modification done to the image (from config or the last updated layer)
"""
LastUpdated: Time
"""
True if the image has a signature associated with it, false otherwise
"""
IsSigned: Boolean
"""
Total size of the files associated with this image (manigest, config, layers)
Total size of the files associated with all images (manifest, config, layers)
"""
Size: String
"""
OS and architecture supported by this image
"""
Platform: OsArch
"""
Vendor associated with this image, the distributing entity, organization or individual
"""
Vendor: String
"""
Integer used to rank search results by relevance
"""
Score: Int
"""
Number of downloads of the manifest of this image
"""
DownloadCount: Int
"""
Information on the layers of this image
Timestamp of the last modification done to the image (from config or the last updated layer)
"""
Layers: [LayerSummary]
LastUpdated: Time
"""
Human-readable description of the software packaged in the image
"""
Description: String
"""
True if the image has a signature associated with it, false otherwise
"""
IsSigned: Boolean
"""
License(s) under which contained software is distributed as an SPDX License Expression
"""
Licenses: String
Licenses: String # The value of the annotation if present, 'unknown' otherwise).
"""
Labels associated with this image
NOTE: currently this field is unused
@ -181,6 +161,10 @@ type ImageSummary {
"""
Title: String
"""
Integer used to rank search results by relevance
"""
Score: Int
"""
URL to get source code for building the image
"""
Source: String
@ -189,6 +173,52 @@ type ImageSummary {
"""
Documentation: String
"""
Vendor associated with this image, the distributing entity, organization or individual
"""
Vendor: String
"""
Contact details of the people or organization responsible for the image
"""
Authors: String
"""
Short summary of the identified CVEs
"""
Vulnerabilities: ImageVulnerabilitySummary
}
"""
Details about a specific version of an image for a certain operating system and architecture.
"""
type ManifestSummary {
"""
Digest of the manifest file associated with this image
"""
Digest: String
"""
Digest of the config file associated with this image
"""
ConfigDigest: String
"""
Timestamp of the last update to an image inside this repository
"""
LastUpdated: Time
"""
Total size of the files associated with this manifest (manifest, config, layers)
"""
Size: String
"""
OS and architecture supported by this image
"""
Platform: Platform
"""
Total numer of image manifest downloads from this repository
"""
DownloadCount: Int
"""
List of layers matching the search criteria
NOTE: the actual search logic for layers is not implemented at the moment
"""
Layers: [LayerSummary]
"""
Information about the history of the specific image, see LayerHistory
"""
History: [LayerHistory]
@ -196,10 +226,6 @@ type ImageSummary {
Short summary of the identified CVEs
"""
Vulnerabilities: ImageVulnerabilitySummary
"""
Contact details of the people or organization responsible for the image
"""
Authors: String
}
"""
@ -235,7 +261,7 @@ type RepoSummary {
"""
List of platforms supported by this repository
"""
Platforms: [OsArch]
Platforms: [Platform]
"""
Vendors associated with this image, the distributing entities, organizations or individuals
"""
@ -371,7 +397,7 @@ type Referrer {
"""
Contains details about the OS and architecture of the image
"""
type OsArch {
type Platform {
"""
The name of the operating system which the image is built to run on,
Should be values listed in the Go Language document https://go.dev/doc/install/source#environment
@ -606,6 +632,8 @@ type Query {
DerivedImageList(
"Image name in the format `repository:tag`"
image: String!,
"Digest of a specific manifest inside the image. When null whole image is considered"
digest: String,
"Sets the parameters of the requested page"
requestedPage: PageInput
): PaginatedImagesResult!
@ -616,6 +644,8 @@ type Query {
BaseImageList(
"Image name in the format `repository:tag`"
image: String!,
"Digest of a specific manifest inside the image. When null whole image is considered"
digest: String,
"Sets the parameters of the requested page"
requestedPage: PageInput
): PaginatedImagesResult!

10
pkg/extensions/search/schema.resolvers.go
View file

@ -104,15 +104,15 @@ func (r *queryResolver) GlobalSearch(ctx context.Context, query string, filter *
}
// DependencyListForImage is the resolver for the DependencyListForImage field.
func (r *queryResolver) DerivedImageList(ctx context.Context, image string, requestedPage *gql_generated.PageInput) (*gql_generated.PaginatedImagesResult, error) {
derivedList, err := derivedImageList(ctx, image, r.repoDB, requestedPage, r.cveInfo, r.log)
func (r *queryResolver) DerivedImageList(ctx context.Context, image string, digest *string, requestedPage *gql_generated.PageInput) (*gql_generated.PaginatedImagesResult, error) {
derivedList, err := derivedImageList(ctx, image, digest, r.repoDB, requestedPage, r.cveInfo, r.log)
return derivedList, err
}
// BaseImageList is the resolver for the BaseImageList field.
func (r *queryResolver) BaseImageList(ctx context.Context, image string, requestedPage *gql_generated.PageInput) (*gql_generated.PaginatedImagesResult, error) {
imageList, err := baseImageList(ctx, image, r.repoDB, requestedPage, r.cveInfo, r.log)
func (r *queryResolver) BaseImageList(ctx context.Context, image string, digest *string, requestedPage *gql_generated.PageInput) (*gql_generated.PaginatedImagesResult, error) {
imageList, err := baseImageList(ctx, image, digest, r.repoDB, requestedPage, r.cveInfo, r.log)
return imageList, err
}
@ -125,7 +125,7 @@ func (r *queryResolver) Image(ctx context.Context, image string) (*gql_generated
return &gql_generated.ImageSummary{}, gqlerror.Errorf("no reference provided")
}
return getImageSummary(ctx, repo, tag, r.repoDB, r.cveInfo, r.log)
return getImageSummary(ctx, repo, tag, nil, r.repoDB, r.cveInfo, r.log)
}
// Referrers is the resolver for the Referrers field.

8
pkg/extensions/sync/sync_test.go
View file

@ -4624,10 +4624,10 @@ func TestSyncImageIndex(t *testing.T) {
err = test.UploadImage(
test.Image{
Manifest: manifest,
Config: config,
Layers: layers,
Tag: manifestDigest.String(),
Manifest: manifest,
Config: config,
Layers: layers,
Reference: manifestDigest.String(),
},
srcBaseURL,
"index")

652
pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go
View file

@ -56,6 +56,11 @@ func NewBoltDBWrapper(params DBParameters) (*DBWrapper, error) {
return err
}
_, err = transaction.CreateBucketIfNotExists([]byte(repodb.IndexDataBucket))
if err != nil {
return err
}
_, err = transaction.CreateBucketIfNotExists([]byte(repodb.RepoMetadataBucket))
if err != nil {
return err
@ -209,6 +214,51 @@ func (bdw DBWrapper) GetManifestMeta(repo string, manifestDigest godigest.Digest
return manifestMetadata, err
}
func (bdw DBWrapper) SetIndexData(indexDigest godigest.Digest, indexMetadata repodb.IndexData) error {
// we make the assumption that the oci layout is consistent and all manifests refferenced inside the
// index are present
err := bdw.DB.Update(func(tx *bolt.Tx) error {
buck := tx.Bucket([]byte(repodb.IndexDataBucket))
imBlob, err := json.Marshal(indexMetadata)
if err != nil {
return errors.Wrapf(err, "repodb: error while calculating blob for manifest with digest %s", indexDigest)
}
err = buck.Put([]byte(indexDigest), imBlob)
if err != nil {
return errors.Wrapf(err, "repodb: error while setting manifest meta with for digest %s", indexDigest)
}
return nil
})
return err
}
func (bdw DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexData, error) {
var indexMetadata repodb.IndexData
err := bdw.DB.View(func(tx *bolt.Tx) error {
buck := tx.Bucket([]byte(repodb.IndexDataBucket))
mmBlob := buck.Get([]byte(indexDigest))
if len(mmBlob) == 0 {
return zerr.ErrManifestMetaNotFound
}
err := json.Unmarshal(mmBlob, &indexMetadata)
if err != nil {
return errors.Wrapf(err, "repodb: error while unmashaling manifest meta for digest %s", indexDigest)
}
return nil
})
return indexMetadata, err
}
func (bdw DBWrapper) SetRepoTag(repo string, tag string, manifestDigest godigest.Digest,
mediaType string,
) error {
@ -622,24 +672,30 @@ func (bdw DBWrapper) DeleteSignature(repo string, signedManifestDigest godigest.
func (bdw DBWrapper) SearchRepos(ctx context.Context, searchText string, filter repodb.Filter,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error) {
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo,
error,
) {
var (
foundRepos = make([]repodb.RepoMetadata, 0)
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
foundindexDataMap = make(map[string]repodb.IndexData)
pageFinder repodb.PageFinder
pageInfo repodb.PageInfo
)
pageFinder, err := repodb.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, repodb.PageInfo{}, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
repodb.PageInfo{}, err
}
err = bdw.DB.View(func(tx *bolt.Tx) error {
err = bdw.DB.View(func(transaction *bolt.Tx) error {
var (
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
repoBuck = tx.Bucket([]byte(repodb.RepoMetadataBucket))
dataBuck = tx.Bucket([]byte(repodb.ManifestDataBucket))
indexDataMap = make(map[string]repodb.IndexData)
repoBuck = transaction.Bucket([]byte(repodb.RepoMetadataBucket))
indexBuck = transaction.Bucket([]byte(repodb.IndexDataBucket))
manifestBuck = transaction.Bucket([]byte(repodb.ManifestDataBucket))
)
cursor := repoBuck.Cursor()
@ -667,47 +723,87 @@ func (bdw DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
isSigned = false
)
for _, descriptor := range repoMeta.Tags {
var manifestMeta repodb.ManifestMetadata
for tag, descriptor := range repoMeta.Tags {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
manifestMeta, manifestDownloaded := manifestMetadataMap[descriptor.Digest]
if !manifestDownloaded {
manifestMetaBlob := dataBuck.Get([]byte(descriptor.Digest))
if manifestMetaBlob == nil {
return zerr.ErrManifestMetaNotFound
}
err := json.Unmarshal(manifestMetaBlob, &manifestMeta)
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest,
manifestMetadataMap, manifestBuck)
if err != nil {
return errors.Wrapf(err, "repodb: error while unmarshaling manifest metadata for digest %s", descriptor.Digest)
return errors.Wrapf(err, "repodb: error fetching manifest meta for manifest with digest %s",
manifestDigest)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return errors.Wrapf(err, "repodb: error collecting filter data for manifest with digest %s",
manifestDigest)
}
repoDownloads += manifestFilterData.DownloadCount
for _, os := range manifestFilterData.OsList {
osSet[os] = true
}
for _, arch := range manifestFilterData.ArchList {
archSet[arch] = true
}
if firstImageChecked || repoLastUpdated.Before(manifestFilterData.LastUpdated) {
repoLastUpdated = manifestFilterData.LastUpdated
firstImageChecked = false
isSigned = manifestFilterData.IsSigned
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
var indexLastUpdated time.Time
indexDigest := descriptor.Digest
indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck)
if err != nil {
return errors.Wrapf(err, "repodb: error fetching index data for index with digest %s",
indexDigest)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return errors.Wrapf(err, "repodb: error while unmashaling index content for %s:%s", repoName, tag)
}
// this also updates manifestMetadataMap
imageFilterData, err := collectImageIndexFilterInfo(indexDigest, repoMeta, indexData, manifestMetadataMap,
manifestBuck)
if err != nil {
return errors.Wrapf(err, "repodb: error collecting filter data for index with digest %s",
indexDigest)
}
for _, arch := range imageFilterData.ArchList {
archSet[arch] = true
}
for _, os := range imageFilterData.OsList {
osSet[os] = true
}
repoDownloads += imageFilterData.DownloadCount
if repoLastUpdated.Before(imageFilterData.LastUpdated) {
repoLastUpdated = indexLastUpdated
isSigned = imageFilterData.IsSigned
}
indexDataMap[indexDigest] = indexData
default:
bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
// get fields related to filtering
var configContent ispec.Image
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
return errors.Wrapf(err, "repodb: error while unmarshaling config content for digest %s", descriptor.Digest)
}
osSet[configContent.OS] = true
archSet[configContent.Architecture] = true
// get fields related to sorting
repoDownloads += repoMeta.Statistics[descriptor.Digest].DownloadCount
imageLastUpdated := common.GetImageLastUpdatedTimestamp(configContent)
if firstImageChecked || repoLastUpdated.Before(imageLastUpdated) {
repoLastUpdated = imageLastUpdated
firstImageChecked = false
isSigned = common.CheckIsSigned(repoMeta.Signatures[descriptor.Digest])
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
}
repoFilterData := repodb.FilterData{
@ -731,40 +827,226 @@ func (bdw DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
foundRepos, pageInfo = pageFinder.Page()
// keep just the manifestMeta we need
// keep just the manifestMeta and indexData we need
for _, repoMeta := range foundRepos {
for _, manifestDigest := range repoMeta.Tags {
foundManifestMetadataMap[manifestDigest.Digest] = manifestMetadataMap[manifestDigest.Digest]
for _, descriptor := range repoMeta.Tags {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
case ispec.MediaTypeImageIndex:
indexData := indexDataMap[descriptor.Digest]
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return err
}
for _, manifestDescriptor := range indexContent.Manifests {
manifestDigest := manifestDescriptor.Digest.String()
foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest]
}
foundindexDataMap[descriptor.Digest] = indexData
default:
bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
}
}
return nil
})
return foundRepos, foundManifestMetadataMap, pageInfo, err
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func fetchManifestMetaWithCheck(repoMeta repodb.RepoMetadata, manifestDigest string,
manifestMetadataMap map[string]repodb.ManifestMetadata, manifestBuck *bolt.Bucket,
) (repodb.ManifestMetadata, error) {
manifestMeta, manifestDownloaded := manifestMetadataMap[manifestDigest]
if !manifestDownloaded {
var manifestData repodb.ManifestData
manifestDataBlob := manifestBuck.Get([]byte(manifestDigest))
if manifestDataBlob == nil {
return repodb.ManifestMetadata{}, zerr.ErrManifestMetaNotFound
}
err := json.Unmarshal(manifestDataBlob, &manifestData)
if err != nil {
return repodb.ManifestMetadata{}, errors.Wrapf(err,
"repodb: error while unmarshaling manifest metadata for digest %s", manifestDigest)
}
manifestMeta = NewManifestMetadata(manifestDigest, repoMeta, manifestData)
}
return manifestMeta, nil
}
func fetchIndexDataWithCheck(indexDigest string, indexDataMap map[string]repodb.IndexData,
indexBuck *bolt.Bucket,
) (repodb.IndexData, error) {
var (
indexData repodb.IndexData
err error
)
indexData, indexExists := indexDataMap[indexDigest]
if !indexExists {
indexDataBlob := indexBuck.Get([]byte(indexDigest))
if indexDataBlob == nil {
return repodb.IndexData{}, zerr.ErrIndexDataNotFount
}
err := json.Unmarshal(indexDataBlob, &indexData)
if err != nil {
return repodb.IndexData{},
errors.Wrapf(err, "repodb: error while unmashaling index data for digest %s", indexDigest)
}
}
return indexData, err
}
func collectImageManifestFilterData(digest string, repoMeta repodb.RepoMetadata,
manifestMeta repodb.ManifestMetadata,
) (repodb.FilterData, error) {
// get fields related to filtering
var (
configContent ispec.Image
osList []string
archList []string
)
err := json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
return repodb.FilterData{},
errors.Wrapf(err, "repodb: error while unmarshaling config content")
}
if configContent.OS != "" {
osList = append(osList, configContent.OS)
}
if configContent.Architecture != "" {
archList = append(archList, configContent.Architecture)
}
return repodb.FilterData{
DownloadCount: repoMeta.Statistics[digest].DownloadCount,
OsList: osList,
ArchList: archList,
LastUpdated: common.GetImageLastUpdatedTimestamp(configContent),
IsSigned: common.CheckIsSigned(repoMeta.Signatures[digest]),
}, nil
}
func collectImageIndexFilterInfo(indexDigest string, repoMeta repodb.RepoMetadata,
indexData repodb.IndexData, manifestMetadataMap map[string]repodb.ManifestMetadata,
manifestBuck *bolt.Bucket,
) (repodb.FilterData, error) {
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return repodb.FilterData{},
errors.Wrapf(err, "repodb: error while unmarshaling index content for digest %s", indexDigest)
}
var (
indexLastUpdated time.Time
firstManifestChecked = false
indexOsList = []string{}
indexArchList = []string{}
)
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest.String(),
manifestMetadataMap, manifestBuck)
if err != nil {
return repodb.FilterData{},
errors.Wrapf(err, "")
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest.String(), repoMeta,
manifestMeta)
if err != nil {
return repodb.FilterData{},
errors.Wrapf(err, "")
}
indexOsList = append(indexOsList, manifestFilterData.OsList...)
indexArchList = append(indexArchList, manifestFilterData.ArchList...)
if !firstManifestChecked || indexLastUpdated.Before(manifestFilterData.LastUpdated) {
indexLastUpdated = manifestFilterData.LastUpdated
firstManifestChecked = true
}
manifestMetadataMap[manifest.Digest.String()] = manifestMeta
}
return repodb.FilterData{
DownloadCount: repoMeta.Statistics[indexDigest].DownloadCount,
LastUpdated: indexLastUpdated,
OsList: indexOsList,
ArchList: indexArchList,
IsSigned: common.CheckIsSigned(repoMeta.Signatures[indexDigest]),
}, nil
}
func NewManifestMetadata(manifestDigest string, repoMeta repodb.RepoMetadata,
manifestData repodb.ManifestData,
) repodb.ManifestMetadata {
manifestMeta := repodb.ManifestMetadata{
ManifestBlob: manifestData.ManifestBlob,
ConfigBlob: manifestData.ConfigBlob,
}
manifestMeta.DownloadCount = repoMeta.Statistics[manifestDigest].DownloadCount
manifestMeta.Signatures = repodb.ManifestSignatures{}
if repoMeta.Signatures[manifestDigest] != nil {
manifestMeta.Signatures = repoMeta.Signatures[manifestDigest]
}
return manifestMeta
}
func (bdw DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error) {
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData,
repodb.PageInfo, error,
) {
var (
foundRepos = make([]repodb.RepoMetadata, 0)
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
indexDataMap = make(map[string]repodb.IndexData)
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
foundindexDataMap = make(map[string]repodb.IndexData)
pageFinder repodb.PageFinder
pageInfo repodb.PageInfo
)
pageFinder, err := repodb.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, repodb.PageInfo{}, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
repodb.PageInfo{}, err
}
err = bdw.DB.View(func(tx *bolt.Tx) error {
var (
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
repoBuck = tx.Bucket([]byte(repodb.RepoMetadataBucket))
dataBuck = tx.Bucket([]byte(repodb.ManifestDataBucket))
cursor = repoBuck.Cursor()
repoBuck = tx.Bucket([]byte(repodb.RepoMetadataBucket))
indexBuck = tx.Bucket([]byte(repodb.IndexDataBucket))
manifestBuck = tx.Bucket([]byte(repodb.ManifestDataBucket))
cursor = repoBuck.Cursor()
)
repoName, repoMetaBlob := cursor.First()
@ -784,46 +1066,69 @@ func (bdw DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
matchedTags := make(map[string]repodb.Descriptor)
// take all manifestMetas
for tag, descriptor := range repoMeta.Tags {
manifestDigest := descriptor.Digest
matchedTags[tag] = descriptor
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
// in case tags reference the same manifest we don't download from DB multiple times
manifestMeta, manifestExists := manifestMetadataMap[manifestDigest]
if !manifestExists {
manifestDataBlob := dataBuck.Get([]byte(manifestDigest))
if manifestDataBlob == nil {
return zerr.ErrManifestMetaNotFound
}
var manifestData repodb.ManifestData
err := json.Unmarshal(manifestDataBlob, &manifestData)
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck)
if err != nil {
return errors.Wrapf(err, "repodb: error while unmashaling manifest metadata for digest %s", manifestDigest)
}
var configContent ispec.Image
if !filter(repoMeta, manifestMeta) {
delete(matchedTags, tag)
err = json.Unmarshal(manifestData.ConfigBlob, &configContent)
continue
}
manifestMetadataMap[manifestDigest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck)
if err != nil {
return errors.Wrapf(err, "repodb: error while unmashaling config for manifest with digest %s", manifestDigest)
return errors.Wrapf(err, "repodb: error while getting index data for digest %s", indexDigest)
}
manifestMeta = repodb.ManifestMetadata{
ConfigBlob: manifestData.ConfigBlob,
ManifestBlob: manifestData.ManifestBlob,
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return errors.Wrapf(err, "repodb: error while unmashaling index content for digest %s", indexDigest)
}
manifestHasBeenMatched := false
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck)
if err != nil {
return errors.Wrapf(err, "repodb: error while getting manifest data for digest %s", manifestDigest)
}
manifestMetadataMap[manifestDigest] = manifestMeta
if filter(repoMeta, manifestMeta) {
manifestHasBeenMatched = true
}
}
if !manifestHasBeenMatched {
delete(matchedTags, tag)
for _, manifest := range indexContent.Manifests {
delete(manifestMetadataMap, manifest.Digest.String())
}
continue
}
indexDataMap[indexDigest] = indexData
default:
bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
if !filter(repoMeta, manifestMeta) {
delete(matchedTags, tag)
continue
}
manifestMetadataMap[manifestDigest] = manifestMeta
}
if len(matchedTags) == 0 {
@ -839,25 +1144,50 @@ func (bdw DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
foundRepos, pageInfo = pageFinder.Page()
// keep just the manifestMeta we need
// keep just the manifestMeta and indexData we need
for _, repoMeta := range foundRepos {
for _, descriptor := range repoMeta.Tags {
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
case ispec.MediaTypeImageIndex:
indexData := indexDataMap[descriptor.Digest]
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return err
}
for _, manifestDescriptor := range indexContent.Manifests {
manifestDigest := manifestDescriptor.Digest.String()
foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest]
}
foundindexDataMap[descriptor.Digest] = indexData
default:
bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
}
}
return nil
})
return foundRepos, foundManifestMetadataMap, pageInfo, err
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func (bdw DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error) {
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
var (
foundRepos = make([]repodb.RepoMetadata, 0)
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
indexDataMap = make(map[string]repodb.IndexData)
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
foundindexDataMap = make(map[string]repodb.IndexData)
pageInfo repodb.PageInfo
pageFinder repodb.PageFinder
@ -865,21 +1195,23 @@ func (bdw DBWrapper) SearchTags(ctx context.Context, searchText string, filter r
pageFinder, err := repodb.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, repodb.PageInfo{}, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
repodb.PageInfo{}, err
}
searchedRepo, searchedTag, err := common.GetRepoTag(searchText)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, repodb.PageInfo{},
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
repodb.PageInfo{},
errors.Wrap(err, "repodb: error while parsing search text, invalid format")
}
err = bdw.DB.View(func(tx *bolt.Tx) error {
var (
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
repoBuck = tx.Bucket([]byte(repodb.RepoMetadataBucket))
dataBuck = tx.Bucket([]byte(repodb.ManifestDataBucket))
cursor = repoBuck.Cursor()
repoBuck = tx.Bucket([]byte(repodb.RepoMetadataBucket))
indexBuck = tx.Bucket([]byte(repodb.IndexDataBucket))
manifestBuck = tx.Bucket([]byte(repodb.ManifestDataBucket))
cursor = repoBuck.Cursor()
)
repoName, repoMetaBlob := cursor.Seek([]byte(searchedRepo))
@ -906,46 +1238,84 @@ func (bdw DBWrapper) SearchTags(ctx context.Context, searchText string, filter r
matchedTags[tag] = descriptor
// in case tags reference the same manifest we don't download from DB multiple times
if manifestMeta, manifestExists := manifestMetadataMap[descriptor.Digest]; manifestExists {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck)
if err != nil {
return errors.Wrapf(err, "repodb: error fetching manifest meta for manifest with digest %s",
manifestDigest)
}
imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return errors.Wrapf(err, "repodb: error collecting filter data for manifest with digest %s",
manifestDigest)
}
if !common.AcceptedByFilter(filter, imageFilterData) {
delete(matchedTags, tag)
continue
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
continue
indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck)
if err != nil {
return errors.Wrapf(err, "repodb: error fetching index data for index with digest %s",
indexDigest)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return errors.Wrapf(err, "repodb: error collecting filter data for index with digest %s",
indexDigest)
}
manifestHasBeenMatched := false
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck)
if err != nil {
return errors.Wrapf(err, "repodb: error fetching from db manifest meta for manifest with digest %s",
manifestDigest)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return errors.Wrapf(err, "repodb: error collecting filter data for manifest with digest %s",
manifestDigest)
}
manifestMetadataMap[manifestDigest] = manifestMeta
if common.AcceptedByFilter(filter, manifestFilterData) {
manifestHasBeenMatched = true
}
}
if !manifestHasBeenMatched {
delete(matchedTags, tag)
for _, manifest := range indexContent.Manifests {
delete(manifestMetadataMap, manifest.Digest.String())
}
continue
}
indexDataMap[indexDigest] = indexData
default:
bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
manifestMetaBlob := dataBuck.Get([]byte(descriptor.Digest))
if manifestMetaBlob == nil {
return zerr.ErrManifestMetaNotFound
}
var manifestMeta repodb.ManifestMetadata
err := json.Unmarshal(manifestMetaBlob, &manifestMeta)
if err != nil {
return errors.Wrapf(err, "repodb: error while unmashaling manifest metadata for digest %s", descriptor.Digest)
}
var configContent ispec.Image
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
return errors.Wrapf(err, "repodb: error while unmashaling config for manifest with digest %s", descriptor.Digest)
}
imageFilterData := repodb.FilterData{
OsList: []string{configContent.OS},
ArchList: []string{configContent.Architecture},
IsSigned: false,
}
if !common.AcceptedByFilter(filter, imageFilterData) {
delete(matchedTags, tag)
delete(manifestMetadataMap, descriptor.Digest)
continue
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
}
if len(matchedTags) == 0 {
@ -962,17 +1332,39 @@ func (bdw DBWrapper) SearchTags(ctx context.Context, searchText string, filter r
foundRepos, pageInfo = pageFinder.Page()
// keep just the manifestMeta we need
// keep just the manifestMeta and indexData we need
for _, repoMeta := range foundRepos {
for _, descriptor := range repoMeta.Tags {
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
case ispec.MediaTypeImageIndex:
indexData := indexDataMap[descriptor.Digest]
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return err
}
for _, manifestDescriptor := range indexContent.Manifests {
manifestDigest := manifestDescriptor.Digest.String()
foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest]
}
foundindexDataMap[descriptor.Digest] = indexData
default:
bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
}
}
return nil
})
return foundRepos, foundManifestMetadataMap, pageInfo, err
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func (bdw *DBWrapper) PatchDB() error {

198
pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go
View file

@ -13,10 +13,12 @@ import (
"zotregistry.io/zot/pkg/meta/repodb"
bolt "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper"
"zotregistry.io/zot/pkg/test"
)
func TestWrapperErrors(t *testing.T) {
Convey("Errors", t, func() {
ctx := context.Background()
tmpDir := t.TempDir()
boltDBParams := bolt.DBParameters{RootDir: tmpDir}
boltdbWrapper, err := bolt.NewBoltDBWrapper(boltDBParams)
@ -300,7 +302,7 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
@ -339,10 +341,10 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo1", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo1", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
_, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo2", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo2", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
@ -378,10 +380,85 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo1", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo1", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("Index Errors", func() {
Convey("Bad index data", func() {
indexDigest := digest.FromString("indexDigest")
err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = setBadIndexData(boltdbWrapper.DB, indexDigest.String())
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("Bad indexBlob in IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = boltdbWrapper.SetIndexData(indexDigest, repodb.IndexData{
IndexBlob: []byte("bad json"),
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("Good index data, bad manifest inside index", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetIndexData(indexDigest, repodb.IndexData{
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetManifestData(manifestDigestFromIndex1, repodb.ManifestData{
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetManifestData(manifestDigestFromIndex2, repodb.ManifestData{
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
})
Convey("SearchTags", func() {
ctx := context.Background()
@ -392,10 +469,10 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, err = boltdbWrapper.SearchTags(ctx, "", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
@ -466,14 +543,117 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo2:", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo2:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo3:", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo3:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("FilterTags Index errors", func() {
Convey("FilterTags bad IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = setBadIndexData(boltdbWrapper.DB, indexDigest.String())
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return true },
repodb.PageInput{},
)
So(err, ShouldNotBeNil)
})
Convey("FilterTags bad indexBlob in IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = boltdbWrapper.SetIndexData(indexDigest, repodb.IndexData{
IndexBlob: []byte("bad json"),
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return true },
repodb.PageInput{},
)
So(err, ShouldNotBeNil)
})
Convey("FilterTags didn't match any index manifest", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetIndexData(indexDigest, repodb.IndexData{
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetManifestData(manifestDigestFromIndex1, repodb.ManifestData{
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("{}"),
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetManifestData(manifestDigestFromIndex2, repodb.ManifestData{
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("{}"),
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return false },
repodb.PageInput{},
)
So(err, ShouldBeNil)
})
})
Convey("Unsuported type", func() {
digest := digest.FromString("digest")
err := boltdbWrapper.SetRepoTag("repo", "tag1", digest, "invalid type") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return true },
repodb.PageInput{},
)
So(err, ShouldBeNil)
})
})
}
func setBadIndexData(dB *bbolt.DB, digest string) error {
return dB.Update(func(tx *bbolt.Tx) error {
indexDataBuck := tx.Bucket([]byte(repodb.IndexDataBucket))
return indexDataBuck.Put([]byte(digest), []byte("bad json"))
})
}

3
pkg/meta/repodb/dynamodb-wrapper/dynamo_internal_test.go
View file

@ -64,6 +64,9 @@ func TestWrapperErrors(t *testing.T) {
err = dynamoWrapper.createManifestDataTable()
So(err, ShouldNotBeNil)
err = dynamoWrapper.createIndexDataTable()
So(err, ShouldNotBeNil)
err = dynamoWrapper.createVersionTable()
So(err, ShouldNotBeNil)
})

363
pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go
View file

@ -12,6 +12,8 @@ import (
"github.com/aws/aws-sdk-go-v2/service/dynamodb"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
guuid "github.com/gofrs/uuid"
"github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/rs/zerolog"
. "github.com/smartystreets/goconvey/convey"
@ -20,6 +22,7 @@ import (
dynamo "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper"
"zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper/iterator"
dynamoParams "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper/params"
"zotregistry.io/zot/pkg/test"
)
func TestIterator(t *testing.T) {
@ -36,6 +39,7 @@ func TestIterator(t *testing.T) {
repoMetaTablename := "RepoMetadataTable" + uuid.String()
manifestDataTablename := "ManifestDataTable" + uuid.String()
versionTablename := "Version" + uuid.String()
indexDataTablename := "IndexDataTable" + uuid.String()
Convey("TestIterator", t, func() {
dynamoWrapper, err := dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{
@ -43,6 +47,7 @@ func TestIterator(t *testing.T) {
Region: region,
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
VersionTablename: versionTablename,
})
So(err, ShouldBeNil)
@ -127,6 +132,7 @@ func TestWrapperErrors(t *testing.T) {
repoMetaTablename := "RepoMetadataTable" + uuid.String()
manifestDataTablename := "ManifestDataTable" + uuid.String()
versionTablename := "Version" + uuid.String()
indexDataTablename := "IndexDataTable" + uuid.String()
ctx := context.Background()
@ -136,6 +142,7 @@ func TestWrapperErrors(t *testing.T) {
Region: region,
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
VersionTablename: versionTablename,
})
So(err, ShouldBeNil)
@ -144,7 +151,7 @@ func TestWrapperErrors(t *testing.T) {
So(dynamoWrapper.ResetRepoMetaTable(), ShouldBeNil) //nolint:contextcheck
Convey("SetManifestData", func() {
dynamoWrapper.ManifestDataTablename = "WRONG table"
dynamoWrapper.ManifestDataTablename = "WRONG tables"
err := dynamoWrapper.SetManifestData("dig", repodb.ManifestData{})
So(err, ShouldNotBeNil)
@ -165,6 +172,21 @@ func TestWrapperErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("GetIndexData", func() {
dynamoWrapper.IndexDataTablename = "WRONG table"
_, err := dynamoWrapper.GetIndexData("dig")
So(err, ShouldNotBeNil)
})
Convey("GetIndexData unmarshal error", func() {
err := setBadIndexData(dynamoWrapper.Client, indexDataTablename, "dig")
So(err, ShouldBeNil)
_, err = dynamoWrapper.GetManifestData("dig")
So(err, ShouldNotBeNil)
})
Convey("SetManifestMeta GetRepoMeta error", func() {
err := setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo1")
So(err, ShouldBeNil)
@ -255,14 +277,6 @@ func TestWrapperErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("IncrementImageDownloads GetManifestMeta error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag", "dig", "")
So(err, ShouldBeNil)
err = dynamoWrapper.IncrementImageDownloads("repo", "tag")
So(err, ShouldNotBeNil)
})
Convey("AddManifestSignature GetRepoMeta error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag", "dig", "")
So(err, ShouldBeNil)
@ -329,22 +343,22 @@ func TestWrapperErrors(t *testing.T) {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchRepos GetManifestMeta error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "notFoundDigest", "") //nolint:contextcheck
err := dynamoWrapper.SetRepoTag("repo", "tag1", "notFoundDigest", ispec.MediaTypeImageManifest) //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchRepos config unmarshal error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig1", "") //nolint:contextcheck
err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig1", ispec.MediaTypeImageManifest) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData("dig1", repodb.ManifestData{ //nolint:contextcheck
@ -353,31 +367,116 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("Unsuported type", func() {
digest := digest.FromString("digest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", digest, "invalid type") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return true },
repodb.PageInput{},
)
So(err, ShouldBeNil)
})
Convey("SearchRepos bad index data", func() {
indexDigest := digest.FromString("indexDigest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchRepos bad indexBlob in IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, repodb.IndexData{ //nolint:contextcheck
IndexBlob: []byte("bad json"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchRepos good index data, bad manifest inside index", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, repodb.IndexData{ //nolint:contextcheck
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex1, repodb.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex2, repodb.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchTags repoMeta unmarshal error", func() {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchTags GetManifestMeta error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "manifestNotFound", "") //nolint:contextcheck
err := dynamoWrapper.SetRepoTag("repo", "tag1", "manifestNotFound", //nolint:contextcheck
ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchTags config unmarshal error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig1", "") //nolint:contextcheck
err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig1", ispec.MediaTypeImageManifest) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData( //nolint:contextcheck
@ -389,16 +488,80 @@ func TestWrapperErrors(t *testing.T) {
)
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchTags bad index data", func() {
indexDigest := digest.FromString("indexDigest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchTags bad indexBlob in IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, repodb.IndexData{ //nolint:contextcheck
IndexBlob: []byte("bad json"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchTags good index data, bad manifest inside index", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, repodb.IndexData{ //nolint:contextcheck
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex1, repodb.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex2, repodb.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("FilterTags repoMeta unmarshal error", func() {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.FilterTags(
_, _, _, _, err = dynamoWrapper.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
return true
@ -410,10 +573,11 @@ func TestWrapperErrors(t *testing.T) {
})
Convey("FilterTags manifestMeta not found", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "manifestNotFound", "") //nolint:contextcheck
err := dynamoWrapper.SetRepoTag("repo", "tag1", "manifestNotFound", //nolint:contextcheck
ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.FilterTags(
_, _, _, _, err = dynamoWrapper.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
return true
@ -425,13 +589,13 @@ func TestWrapperErrors(t *testing.T) {
})
Convey("FilterTags manifestMeta unmarshal error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig", "") //nolint:contextcheck
err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig", ispec.MediaTypeImageManifest) //nolint:contextcheck
So(err, ShouldBeNil)
err = setBadManifestData(dynamoWrapper.Client, manifestDataTablename, "dig") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.FilterTags(
_, _, _, _, err = dynamoWrapper.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
return true
@ -442,26 +606,130 @@ func TestWrapperErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("FilterTags config unmarshal error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig1", "") //nolint:contextcheck
Convey("FilterTags bad IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData("dig1", repodb.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("bad json"),
err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return true },
repodb.PageInput{},
)
So(err, ShouldNotBeNil)
})
Convey("FilterTags bad indexBlob in IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, repodb.IndexData{ //nolint:contextcheck
IndexBlob: []byte("bad json"),
})
So(err, ShouldBeNil)
_, _, _, err = dynamoWrapper.FilterTags(
ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
return true
},
_, _, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return true },
repodb.PageInput{},
)
So(err, ShouldNotBeNil)
})
Convey("FilterTags didn't match any index manifest", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, repodb.IndexData{ //nolint:contextcheck
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex1, repodb.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("{}"),
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex2, repodb.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("{}"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return false },
repodb.PageInput{},
)
So(err, ShouldBeNil)
})
})
Convey("NewDynamoDBWrapper errors", t, func() {
_, err := dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck
Endpoint: endpoint,
Region: region,
RepoMetaTablename: "",
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
VersionTablename: versionTablename,
})
So(err, ShouldNotBeNil)
_, err = dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck
Endpoint: endpoint,
Region: region,
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: "",
IndexDataTablename: indexDataTablename,
VersionTablename: versionTablename,
})
So(err, ShouldNotBeNil)
_, err = dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck
Endpoint: endpoint,
Region: region,
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: "",
VersionTablename: versionTablename,
})
So(err, ShouldNotBeNil)
_, err = dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck
Endpoint: endpoint,
Region: region,
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
VersionTablename: "",
})
So(err, ShouldNotBeNil)
_, err = dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck
Endpoint: endpoint,
Region: region,
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
VersionTablename: versionTablename,
})
So(err, ShouldBeNil)
})
}
@ -490,6 +758,31 @@ func setBadManifestData(client *dynamodb.Client, manifestDataTableName, digest s
return err
}
func setBadIndexData(client *dynamodb.Client, indexDataTableName, digest string) error {
mdAttributeValue, err := attributevalue.Marshal("string")
if err != nil {
return err
}
_, err = client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{
ExpressionAttributeNames: map[string]string{
"#ID": "IndexData",
},
ExpressionAttributeValues: map[string]types.AttributeValue{
":IndexData": mdAttributeValue,
},
Key: map[string]types.AttributeValue{
"IndexDigest": &types.AttributeValueMemberS{
Value: digest,
},
},
TableName: aws.String(indexDataTableName),
UpdateExpression: aws.String("SET #ID = :IndexData"),
})
return err
}
func setBadRepoMeta(client *dynamodb.Client, repoMetadataTableName, repoName string) error {
repoAttributeValue, err := attributevalue.Marshal("string")
if err != nil {

633
pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go
View file

@ -30,6 +30,7 @@ import (
type DBWrapper struct {
Client *dynamodb.Client
RepoMetaTablename string
IndexDataTablename string
ManifestDataTablename string
VersionTablename string
Patches []func(client *dynamodb.Client, tableNames map[string]string) error
@ -60,6 +61,7 @@ func NewDynamoDBWrapper(params dynamoParams.DBDriverParameters) (*DBWrapper, err
Client: dynamodb.NewFromConfig(cfg),
RepoMetaTablename: params.RepoMetaTablename,
ManifestDataTablename: params.ManifestDataTablename,
IndexDataTablename: params.IndexDataTablename,
VersionTablename: params.VersionTablename,
Patches: version.GetDynamoDBPatches(),
Log: log.Logger{Logger: zerolog.New(os.Stdout)},
@ -80,6 +82,11 @@ func NewDynamoDBWrapper(params dynamoParams.DBDriverParameters) (*DBWrapper, err
return nil, err
}
err = dynamoWrapper.createIndexDataTable()
if err != nil {
return nil, err
}
// Using the Config value, create the DynamoDB client
return &dynamoWrapper, nil
}
@ -248,6 +255,58 @@ func (dwr DBWrapper) GetRepoStars(repo string) (int, error) {
return repoMeta.Stars, nil
}
func (dwr DBWrapper) SetIndexData(indexDigest godigest.Digest, indexData repodb.IndexData) error {
indexAttributeValue, err := attributevalue.Marshal(indexData)
if err != nil {
return err
}
_, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{
ExpressionAttributeNames: map[string]string{
"#ID": "IndexData",
},
ExpressionAttributeValues: map[string]types.AttributeValue{
":IndexData": indexAttributeValue,
},
Key: map[string]types.AttributeValue{
"IndexDigest": &types.AttributeValueMemberS{
Value: indexDigest.String(),
},
},
TableName: aws.String(dwr.IndexDataTablename),
UpdateExpression: aws.String("SET #ID = :IndexData"),
})
return err
}
func (dwr DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexData, error) {
resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{
TableName: aws.String(dwr.IndexDataTablename),
Key: map[string]types.AttributeValue{
"IndexDigest": &types.AttributeValueMemberS{
Value: indexDigest.String(),
},
},
})
if err != nil {
return repodb.IndexData{}, err
}
if resp.Item == nil {
return repodb.IndexData{}, zerr.ErrRepoMetaNotFound
}
var indexData repodb.IndexData
err = attributevalue.Unmarshal(resp.Item["IndexData"], &indexData)
if err != nil {
return repodb.IndexData{}, err
}
return indexData, nil
}
func (dwr DBWrapper) SetRepoTag(repo string, tag string, manifestDigest godigest.Digest, mediaType string) error {
if err := common.ValidateRepoTagInput(repo, tag, manifestDigest); err != nil {
return err
@ -377,7 +436,7 @@ func (dwr DBWrapper) IncrementImageDownloads(repo string, reference string) erro
return err
}
manifestDigest := reference
descriptorDigest := reference
if !common.ReferenceIsDigest(reference) {
// search digest for tag
@ -387,19 +446,14 @@ func (dwr DBWrapper) IncrementImageDownloads(repo string, reference string) erro
return zerr.ErrManifestMetaNotFound
}
manifestDigest = descriptor.Digest
descriptorDigest = descriptor.Digest
}
manifestMeta, err := dwr.GetManifestMeta(repo, godigest.Digest(manifestDigest))
if err != nil {
return err
}
manifestStatistics := repoMeta.Statistics[descriptorDigest]
manifestStatistics.DownloadCount++
repoMeta.Statistics[descriptorDigest] = manifestStatistics
manifestMeta.DownloadCount++
err = dwr.SetManifestMeta(repo, godigest.Digest(manifestDigest), manifestMeta)
return err
return dwr.setRepoMeta(repo, repoMeta)
}
func (dwr DBWrapper) AddManifestSignature(repo string, signedManifestDigest godigest.Digest,
@ -531,11 +585,10 @@ func (dwr DBWrapper) GetMultipleRepoMeta(ctx context.Context,
func (dwr DBWrapper) SearchRepos(ctx context.Context, searchText string, filter repodb.Filter,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error) {
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
var (
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
indexDataMap = make(map[string]repodb.IndexData)
repoMetaAttributeIterator iterator.AttributesIterator
pageFinder repodb.PageFinder
pageInfo repodb.PageInfo
@ -547,7 +600,8 @@ func (dwr DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
pageFinder, err := repodb.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx)
@ -555,14 +609,16 @@ func (dwr DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
// log
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
var repoMeta repodb.RepoMetadata
err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
@ -581,43 +637,84 @@ func (dwr DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
)
for _, descriptor := range repoMeta.Tags {
var manifestMeta repodb.ManifestMetadata
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
manifestMeta, manifestDownloaded := manifestMetadataMap[descriptor.Digest]
if !manifestDownloaded {
manifestMeta, err = dwr.GetManifestMeta(repoMeta.Name, godigest.Digest(descriptor.Digest)) //nolint:contextcheck
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo,
errors.Wrapf(err, "repodb: error while unmarshaling manifest metadata for digest %s", descriptor.Digest)
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "")
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "")
}
repoDownloads += manifestFilterData.DownloadCount
for _, os := range manifestFilterData.OsList {
osSet[os] = true
}
for _, arch := range manifestFilterData.ArchList {
archSet[arch] = true
}
if firstImageChecked || repoLastUpdated.Before(manifestFilterData.LastUpdated) {
repoLastUpdated = manifestFilterData.LastUpdated
firstImageChecked = false
isSigned = manifestFilterData.IsSigned
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
var indexLastUpdated time.Time
indexDigest := descriptor.Digest
indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "")
}
// this also updates manifestMetadataMap
imageFilterData, err := dwr.collectImageIndexFilterInfo(indexDigest, repoMeta, indexData, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "")
}
for _, arch := range imageFilterData.ArchList {
archSet[arch] = true
}
for _, os := range imageFilterData.OsList {
osSet[os] = true
}
repoDownloads += imageFilterData.DownloadCount
if repoLastUpdated.Before(imageFilterData.LastUpdated) {
repoLastUpdated = indexLastUpdated
isSigned = imageFilterData.IsSigned
}
indexDataMap[indexDigest] = indexData
default:
dwr.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
// get fields related to filtering
var configContent ispec.Image
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo,
errors.Wrapf(err, "repodb: error while unmarshaling config content for digest %s", descriptor.Digest)
}
osSet[configContent.OS] = true
archSet[configContent.Architecture] = true
// get fields related to sorting
repoDownloads += repoMeta.Statistics[descriptor.Digest].DownloadCount
imageLastUpdated := common.GetImageLastUpdatedTimestamp(configContent)
if firstImageChecked || repoLastUpdated.Before(imageLastUpdated) {
repoLastUpdated = imageLastUpdated
firstImageChecked = false
isSigned = common.CheckIsSigned(manifestMeta.Signatures)
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
}
repoFilterData := repodb.FilterData{
@ -641,22 +738,145 @@ func (dwr DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
foundRepos, pageInfo := pageFinder.Page()
// keep just the manifestMeta we need
for _, repoMeta := range foundRepos {
for _, descriptor := range repoMeta.Tags {
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
foundManifestMetadataMap, foundindexDataMap, err := filterFoundData(foundRepos, manifestMetadataMap, indexDataMap)
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func (dwr DBWrapper) fetchManifestMetaWithCheck(repoName string, manifestDigest string,
manifestMetadataMap map[string]repodb.ManifestMetadata,
) (repodb.ManifestMetadata, error) {
var (
manifestMeta repodb.ManifestMetadata
err error
)
manifestMeta, manifestDownloaded := manifestMetadataMap[manifestDigest]
if !manifestDownloaded {
manifestMeta, err = dwr.GetManifestMeta(repoName, godigest.Digest(manifestDigest)) //nolint:contextcheck
if err != nil {
return repodb.ManifestMetadata{}, err
}
}
return foundRepos, foundManifestMetadataMap, pageInfo, err
return manifestMeta, nil
}
func collectImageManifestFilterData(digest string, repoMeta repodb.RepoMetadata,
manifestMeta repodb.ManifestMetadata,
) (repodb.FilterData, error) {
// get fields related to filtering
var (
configContent ispec.Image
osList []string
archList []string
)
err := json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
return repodb.FilterData{},
errors.Wrapf(err, "repodb: error while unmarshaling config content")
}
if configContent.OS != "" {
osList = append(osList, configContent.OS)
}
if configContent.Architecture != "" {
archList = append(archList, configContent.Architecture)
}
return repodb.FilterData{
DownloadCount: repoMeta.Statistics[digest].DownloadCount,
OsList: osList,
ArchList: archList,
LastUpdated: common.GetImageLastUpdatedTimestamp(configContent),
IsSigned: common.CheckIsSigned(repoMeta.Signatures[digest]),
}, nil
}
func (dwr DBWrapper) fetchIndexDataWithCheck(indexDigest string, indexDataMap map[string]repodb.IndexData,
) (repodb.IndexData, error) {
var (
indexData repodb.IndexData
err error
)
indexData, indexExists := indexDataMap[indexDigest]
if !indexExists {
indexData, err = dwr.GetIndexData(godigest.Digest(indexDigest)) //nolint:contextcheck
if err != nil {
return repodb.IndexData{},
errors.Wrapf(err, "repodb: error while unmarshaling index data for digest %s", indexDigest)
}
}
return indexData, err
}
func (dwr DBWrapper) collectImageIndexFilterInfo(indexDigest string, repoMeta repodb.RepoMetadata,
indexData repodb.IndexData, manifestMetadataMap map[string]repodb.ManifestMetadata,
) (repodb.FilterData, error) {
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return repodb.FilterData{},
errors.Wrapf(err, "repodb: error while unmarshaling index content for digest %s", indexDigest)
}
var (
indexLastUpdated time.Time
firstManifestChecked = false
indexOsList = []string{}
indexArchList = []string{}
)
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest.String(),
manifestMetadataMap)
if err != nil {
return repodb.FilterData{},
errors.Wrapf(err, "")
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest.String(), repoMeta,
manifestMeta)
if err != nil {
return repodb.FilterData{},
errors.Wrapf(err, "")
}
indexOsList = append(indexOsList, manifestFilterData.OsList...)
indexArchList = append(indexArchList, manifestFilterData.ArchList...)
if !firstManifestChecked || indexLastUpdated.Before(manifestFilterData.LastUpdated) {
indexLastUpdated = manifestFilterData.LastUpdated
firstManifestChecked = true
}
manifestMetadataMap[manifest.Digest.String()] = manifestMeta
}
return repodb.FilterData{
DownloadCount: repoMeta.Statistics[indexDigest].DownloadCount,
LastUpdated: indexLastUpdated,
OsList: indexOsList,
ArchList: indexArchList,
IsSigned: common.CheckIsSigned(repoMeta.Signatures[indexDigest]),
}, nil
}
func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error) {
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
var (
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
indexDataMap = make(map[string]repodb.IndexData)
pageFinder repodb.PageFinder
repoMetaAttributeIterator iterator.AttributesIterator
pageInfo repodb.PageInfo
@ -668,7 +888,8 @@ func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
pageFinder, err := repodb.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx)
@ -676,14 +897,16 @@ func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
// log
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
var repoMeta repodb.RepoMetadata
err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
@ -692,36 +915,80 @@ func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
matchedTags := make(map[string]repodb.Descriptor)
// take all manifestMetas
for tag, descriptor := range repoMeta.Tags {
manifestDigest := descriptor.Digest
matchedTags[tag] = descriptor
// in case tags reference the same manifest we don't download from DB multiple times
manifestMeta, manifestExists := manifestMetadataMap[manifestDigest]
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
if !manifestExists {
manifestMeta, err := dwr.GetManifestMeta(repoMeta.Name, godigest.Digest(manifestDigest)) //nolint:contextcheck
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo,
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "repodb: error while unmashaling manifest metadata for digest %s", manifestDigest)
}
var configContent ispec.Image
if !filter(repoMeta, manifestMeta) {
delete(matchedTags, tag)
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo,
errors.Wrapf(err, "repodb: error while unmashaling config for manifest with digest %s", manifestDigest)
continue
}
manifestMetadataMap[manifestDigest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "repodb: error while getting index data for digest %s", indexDigest)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "repodb: error while unmashaling index content for digest %s", indexDigest)
}
manifestHasBeenMatched := false
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "repodb: error while getting manifest data for digest %s", manifestDigest)
}
manifestMetadataMap[manifestDigest] = manifestMeta
if filter(repoMeta, manifestMeta) {
manifestHasBeenMatched = true
}
}
if !manifestHasBeenMatched {
delete(matchedTags, tag)
for _, manifest := range indexContent.Manifests {
delete(manifestMetadataMap, manifest.Digest.String())
}
continue
}
indexDataMap[indexDigest] = indexData
default:
dwr.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
if !filter(repoMeta, manifestMeta) {
delete(matchedTags, tag)
continue
}
manifestMetadataMap[manifestDigest] = manifestMeta
}
if len(matchedTags) == 0 {
@ -737,22 +1004,17 @@ func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
foundRepos, pageInfo := pageFinder.Page()
// keep just the manifestMeta we need
for _, repoMeta := range foundRepos {
for _, descriptor := range repoMeta.Tags {
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
}
}
foundManifestMetadataMap, foundindexDataMap, err := filterFoundData(foundRepos, manifestMetadataMap, indexDataMap)
return foundRepos, foundManifestMetadataMap, pageInfo, err
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func (dwr DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error) {
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
var (
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
indexDataMap = make(map[string]repodb.IndexData)
repoMetaAttributeIterator = iterator.NewBaseDynamoAttributesIterator(
dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log,
)
@ -763,12 +1025,14 @@ func (dwr DBWrapper) SearchTags(ctx context.Context, searchText string, filter r
pageFinder, err := repodb.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
searchedRepo, searchedTag, err := common.GetRepoTag(searchText)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo,
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrap(err, "repodb: error while parsing search text, invalid format")
}
@ -777,14 +1041,16 @@ func (dwr DBWrapper) SearchTags(ctx context.Context, searchText string, filter r
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
// log
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
var repoMeta repodb.RepoMetadata
err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo, err
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
@ -801,41 +1067,92 @@ func (dwr DBWrapper) SearchTags(ctx context.Context, searchText string, filter r
matchedTags[tag] = descriptor
// in case tags reference the same manifest we don't download from DB multiple times
if manifestMeta, manifestExists := manifestMetadataMap[descriptor.Digest]; manifestExists {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "repodb: error while unmashaling manifest metadata for digest %s", descriptor.Digest)
}
imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "")
}
if !common.AcceptedByFilter(filter, imageFilterData) {
delete(matchedTags, tag)
continue
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
continue
indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "")
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "repodb: error while unmashaling index content for digest %s", indexDigest)
}
manifestHasBeenMatched := false
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "")
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
errors.Wrapf(err, "")
}
manifestMetadataMap[manifestDigest] = manifestMeta
if common.AcceptedByFilter(filter, manifestFilterData) {
manifestHasBeenMatched = true
}
}
if !manifestHasBeenMatched {
delete(matchedTags, tag)
for _, manifest := range indexContent.Manifests {
delete(manifestMetadataMap, manifest.Digest.String())
}
continue
}
indexDataMap[indexDigest] = indexData
default:
dwr.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
manifestMeta, err := dwr.GetManifestMeta(repoMeta.Name, godigest.Digest(descriptor.Digest)) //nolint:contextcheck
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo,
errors.Wrapf(err, "repodb: error while unmashaling manifest metadata for digest %s", descriptor.Digest)
}
var configContent ispec.Image
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, pageInfo,
errors.Wrapf(err, "repodb: error while unmashaling config for manifest with digest %s", descriptor.Digest)
}
imageFilterData := repodb.FilterData{
OsList: []string{configContent.OS},
ArchList: []string{configContent.Architecture},
IsSigned: false,
}
if !common.AcceptedByFilter(filter, imageFilterData) {
delete(matchedTags, tag)
delete(manifestMetadataMap, descriptor.Digest)
continue
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
}
if len(matchedTags) == 0 {
@ -852,14 +1169,49 @@ func (dwr DBWrapper) SearchTags(ctx context.Context, searchText string, filter r
foundRepos, pageInfo := pageFinder.Page()
foundManifestMetadataMap, foundindexDataMap, err := filterFoundData(foundRepos, manifestMetadataMap, indexDataMap)
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func filterFoundData(foundRepos []repodb.RepoMetadata, manifestMetadataMap map[string]repodb.ManifestMetadata,
indexDataMap map[string]repodb.IndexData,
) (map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, error) {
var (
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
foundindexDataMap = make(map[string]repodb.IndexData)
)
// keep just the manifestMeta we need
for _, repoMeta := range foundRepos {
for _, descriptor := range repoMeta.Tags {
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
case ispec.MediaTypeImageIndex:
indexData := indexDataMap[descriptor.Digest]
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
errors.Wrapf(err, "repodb: error while getting manifest data for digest %s", descriptor.Digest)
}
for _, manifestDescriptor := range indexContent.Manifests {
manifestDigest := manifestDescriptor.Digest.String()
foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest]
}
foundindexDataMap[descriptor.Digest] = indexData
default:
}
}
}
return foundRepos, foundManifestMetadataMap, pageInfo, err
return foundManifestMetadataMap, foundindexDataMap, nil
}
func (dwr *DBWrapper) PatchDB() error {
@ -1008,6 +1360,31 @@ func (dwr DBWrapper) createManifestDataTable() error {
return dwr.waitTableToBeCreated(dwr.ManifestDataTablename)
}
func (dwr DBWrapper) createIndexDataTable() error {
_, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{
TableName: aws.String(dwr.IndexDataTablename),
AttributeDefinitions: []types.AttributeDefinition{
{
AttributeName: aws.String("IndexDigest"),
AttributeType: types.ScalarAttributeTypeS,
},
},
KeySchema: []types.KeySchemaElement{
{
AttributeName: aws.String("IndexDigest"),
KeyType: types.KeyTypeHash,
},
},
BillingMode: types.BillingModePayPerRequest,
})
if err != nil && strings.Contains(err.Error(), "Table already exists") {
return nil
}
return dwr.waitTableToBeCreated(dwr.IndexDataTablename)
}
func (dwr *DBWrapper) createVersionTable() error {
_, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{
TableName: aws.String(dwr.VersionTablename),

3
pkg/meta/repodb/dynamodb-wrapper/params/parameters.go
View file

@ -1,5 +1,6 @@
package params
type DBDriverParameters struct {
Endpoint, Region, RepoMetaTablename, ManifestDataTablename, VersionTablename string
Endpoint, Region, RepoMetaTablename, ManifestDataTablename, IndexDataTablename,
VersionTablename string
}

26
pkg/meta/repodb/repodb.go
View file

@ -2,6 +2,7 @@ package repodb
import (
"context"
"time"
godigest "github.com/opencontainers/go-digest"
)
@ -9,6 +10,7 @@ import (
// MetadataDB.
const (
ManifestDataBucket = "ManifestData"
IndexDataBucket = "IndexData"
UserMetadataBucket = "UserMeta"
RepoMetadataBucket = "RepoMetadata"
VersionBucket = "Version"
@ -59,6 +61,12 @@ type RepoDB interface { //nolint:interfacebloat
// GetManifestMeta sets ManifestMetadata for a given manifest in the database
SetManifestMeta(repo string, manifestDigest godigest.Digest, mm ManifestMetadata) error
// SetIndexData sets indexData for a given index in the database
SetIndexData(digest godigest.Digest, indexData IndexData) error
// GetIndexData returns indexData for a given Index from the database
GetIndexData(indexDigest godigest.Digest) (IndexData, error)
// IncrementManifestDownloads adds 1 to the download count of a manifest
IncrementImageDownloads(repo string, reference string) error
@ -70,15 +78,15 @@ type RepoDB interface { //nolint:interfacebloat
// SearchRepos searches for repos given a search string
SearchRepos(ctx context.Context, searchText string, filter Filter, requestedPage PageInput) (
[]RepoMetadata, map[string]ManifestMetadata, PageInfo, error)
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, PageInfo, error)
// SearchTags searches for images(repo:tag) given a search string
SearchTags(ctx context.Context, searchText string, filter Filter, requestedPage PageInput) (
[]RepoMetadata, map[string]ManifestMetadata, PageInfo, error)
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, PageInfo, error)
// FilterTags filters for images given a filter function
FilterTags(ctx context.Context, filter FilterFunc,
requestedPage PageInput) ([]RepoMetadata, map[string]ManifestMetadata, PageInfo, error)
requestedPage PageInput) ([]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, PageInfo, error)
PatchDB() error
}
@ -90,6 +98,10 @@ type ManifestMetadata struct {
Signatures ManifestSignatures
}
type IndexData struct {
IndexBlob []byte
}
type ManifestData struct {
ManifestBlob []byte
ConfigBlob []byte
@ -163,7 +175,9 @@ type Filter struct {
}
type FilterData struct {
OsList []string
ArchList []string
IsSigned bool
DownloadCount int
LastUpdated time.Time
OsList []string
ArchList []string
IsSigned bool
}

553
pkg/meta/repodb/repodb_test.go
View file

File diff suppressed because it is too large Load diff

1
pkg/meta/repodb/repodbfactory/repodb_factory_test.go
View file

@ -19,6 +19,7 @@ func TestCreateDynamo(t *testing.T) {
Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"),
RepoMetaTablename: "RepoMetadataTable",
ManifestDataTablename: "ManifestDataTable",
IndexDataTablename: "IndexDataTable",
VersionTablename: "Version",
Region: "us-east-2",
}

93
pkg/meta/repodb/sync_repodb.go
View file

@ -74,12 +74,6 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
for _, manifest := range indexContent.Manifests {
tag, hasTag := manifest.Annotations[ispec.AnnotationRefName]
if !hasTag {
log.Warn().Msgf("sync-repo: image without tag found, will not be synced into RepoDB")
continue
}
manifestMetaIsPresent, err := isManifestMetaPresent(repo, manifest, repoDB)
if err != nil {
log.Error().Err(err).Msgf("sync-repo: error checking manifestMeta in RepoDB")
@ -87,7 +81,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
return err
}
if manifestMetaIsPresent {
if manifestMetaIsPresent && hasTag {
err = repoDB.SetRepoTag(repo, tag, manifest.Digest, manifest.MediaType)
if err != nil {
log.Error().Err(err).Msgf("sync-repo: failed to set repo tag for %s:%s", repo, tag)
@ -131,31 +125,16 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
continue
}
manifestData, err := NewManifestData(repo, manifestBlob, storeController)
if err != nil {
log.Error().Err(err).Msgf("sync-repo: failed to create manifest data for image %s:%s manifest digest %s ",
repo, tag, manifest.Digest.String())
reference := tag
return err
if tag == "" {
reference = manifest.Digest.String()
}
err = repoDB.SetManifestMeta(repo, manifest.Digest, ManifestMetadata{
ManifestBlob: manifestData.ManifestBlob,
ConfigBlob: manifestData.ConfigBlob,
DownloadCount: 0,
Signatures: ManifestSignatures{},
})
err = SetMetadataFromInput(repo, reference, manifest.MediaType, manifest.Digest, manifestBlob,
storeController, repoDB, log)
if err != nil {
log.Error().Err(err).Msgf("sync-repo: failed to set manifest meta for image %s:%s manifest digest %s ",
repo, tag, manifest.Digest.String())
return err
}
err = repoDB.SetRepoTag(repo, tag, manifest.Digest, manifest.MediaType)
if err != nil {
log.Error().Err(err).Msgf("sync-repo: failed to repo tag for repo %s and tag %s",
repo, tag)
log.Error().Err(err).Msgf("sync-repo: failed to set metadata for %s:%s", repo, tag)
return err
}
@ -271,3 +250,61 @@ func NewManifestData(repoName string, manifestBlob []byte, storeController stora
return manifestData, nil
}
func NewIndexData(repoName string, indexBlob []byte, storeController storage.StoreController,
) IndexData {
indexData := IndexData{}
indexData.IndexBlob = indexBlob
return indexData
}
// SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag
// (in case the reference is a tag). The function expects image manifests and indexes (multi arch images).
func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Digest, descriptorBlob []byte,
storeController storage.StoreController, repoDB RepoDB, log log.Logger,
) error {
switch mediaType {
case ispec.MediaTypeImageManifest:
imageData, err := NewManifestData(repo, descriptorBlob, storeController)
if err != nil {
return err
}
err = repoDB.SetManifestData(digest, imageData)
if err != nil {
log.Error().Err(err).Msg("repodb: error while putting manifest meta")
return err
}
case ispec.MediaTypeImageIndex:
indexData := NewIndexData(repo, descriptorBlob, storeController)
err := repoDB.SetIndexData(digest, indexData)
if err != nil {
log.Error().Err(err).Msg("repodb: error while putting index data")
return err
}
}
if refferenceIsDigest(reference) {
return nil
}
err := repoDB.SetRepoTag(repo, reference, digest, mediaType)
if err != nil {
log.Error().Err(err).Msg("repodb: error while putting repo meta")
return err
}
return nil
}
func refferenceIsDigest(reference string) bool {
_, err := godigest.Parse(reference)
return err == nil
}

66
pkg/meta/repodb/sync_repodb_test.go
View file

@ -298,10 +298,10 @@ func TestSyncRepoDBWithStorage(t *testing.T) {
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
Tag: fmt.Sprintf("tag%d", i),
Config: config,
Layers: layers,
Manifest: manifest,
Reference: fmt.Sprintf("tag%d", i),
},
repo,
storeController)
@ -322,10 +322,10 @@ func TestSyncRepoDBWithStorage(t *testing.T) {
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
Tag: signatureTag,
Config: config,
Layers: layers,
Manifest: manifest,
Reference: signatureTag,
},
repo,
storeController)
@ -398,10 +398,10 @@ func TestSyncRepoDBWithStorage(t *testing.T) {
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
Tag: "tag1",
Config: config,
Layers: layers,
Manifest: manifest,
Reference: "tag1",
},
repo,
storeController)
@ -420,10 +420,10 @@ func TestSyncRepoDBWithStorage(t *testing.T) {
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
Tag: signatureTag,
Config: config,
Layers: layers,
Manifest: manifest,
Reference: signatureTag,
},
repo,
storeController)
@ -470,10 +470,10 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
Tag: fmt.Sprintf("tag%d", i),
Config: config,
Layers: layers,
Manifest: manifest,
Reference: fmt.Sprintf("tag%d", i),
},
repo,
storeController)
@ -494,10 +494,10 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
Tag: signatureTag,
Config: config,
Layers: layers,
Manifest: manifest,
Reference: signatureTag,
},
repo,
storeController)
@ -531,6 +531,7 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
Region: "us-east-2",
RepoMetaTablename: "RepoMetadataTable",
ManifestDataTablename: "ManifestDataTable",
IndexDataTablename: "IndexDataTable",
VersionTablename: "Version",
})
So(err, ShouldBeNil)
@ -580,10 +581,10 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
Tag: "tag1",
Config: config,
Layers: layers,
Manifest: manifest,
Reference: "tag1",
},
repo,
storeController)
@ -602,10 +603,10 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
Tag: signatureTag,
Config: config,
Layers: layers,
Manifest: manifest,
Reference: signatureTag,
},
repo,
storeController)
@ -617,6 +618,7 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
Region: "us-east-2",
RepoMetaTablename: "RepoMetadataTable",
ManifestDataTablename: "ManifestDataTable",
IndexDataTablename: "IndexDataTable",
VersionTablename: "Version",
})
So(err, ShouldBeNil)

45
pkg/meta/repodb/update/update.go
View file

@ -51,7 +51,7 @@ func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest,
metadataSuccessfullySet = false
}
} else {
err := SetMetadataFromInput(name, reference, mediaType, digest, body,
err := repodb.SetMetadataFromInput(name, reference, mediaType, digest, body,
storeController, repoDB, log)
if err != nil {
metadataSuccessfullySet = false
@ -160,46 +160,3 @@ func OnGetManifest(name, reference string, digest godigest.Digest, body []byte,
return nil
}
// SetMetadataFromInput receives raw information about the manifest pushed and tries to set manifest metadata
// and update repo metadata by adding the current tag (in case the reference is a tag).
// The function expects image manifest.
func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Digest, manifestBlob []byte,
storeController storage.StoreController, repoDB repodb.RepoDB, log log.Logger,
) error {
imageMetadata, err := repodb.NewManifestData(repo, manifestBlob, storeController)
if err != nil {
return err
}
err = repoDB.SetManifestMeta(repo, digest, repodb.ManifestMetadata{
ManifestBlob: imageMetadata.ManifestBlob,
ConfigBlob: imageMetadata.ConfigBlob,
DownloadCount: 0,
Signatures: repodb.ManifestSignatures{},
})
if err != nil {
log.Error().Err(err).Msg("repodb: error while putting image meta")
return err
}
if refferenceIsDigest(reference) {
return nil
}
err = repoDB.SetRepoTag(repo, reference, digest, mediaType)
if err != nil {
log.Error().Err(err).Msg("repodb: error while putting repo meta")
return err
}
return nil
}
func refferenceIsDigest(reference string) bool {
_, err := godigest.Parse(reference)
return err == nil
}

35
pkg/meta/repodb/update/update_test.go
View file

@ -14,6 +14,7 @@ import (
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
bolt_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper"
repoDBUpdate "zotregistry.io/zot/pkg/meta/repodb/update"
"zotregistry.io/zot/pkg/storage"
@ -42,8 +43,12 @@ func TestOnUpdateManifest(t *testing.T) {
config, layers, manifest, err := test.GetRandomImageComponents(100)
So(err, ShouldBeNil)
err = test.WriteImageToFileSystem(test.Image{Config: config, Manifest: manifest, Layers: layers, Tag: "tag1"},
"repo", storeController)
err = test.WriteImageToFileSystem(
test.Image{
Config: config, Manifest: manifest, Layers: layers, Reference: "tag1",
},
"repo",
storeController)
So(err, ShouldBeNil)
manifestBlob, err := json.Marshal(manifest)
@ -59,6 +64,26 @@ func TestOnUpdateManifest(t *testing.T) {
So(repoMeta.Tags, ShouldContainKey, "tag1")
})
Convey("metadataSuccessfullySet is false", t, func() {
rootDir := t.TempDir()
storeController := storage.StoreController{}
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
storeController.DefaultStore = local.NewImageStore(rootDir, true, 1*time.Second,
true, true, log, metrics, nil, nil,
)
repoDB := mocks.RepoDBMock{
SetManifestDataFn: func(manifestDigest godigest.Digest, mm repodb.ManifestData) error {
return ErrTestError
},
}
err := repoDBUpdate.OnUpdateManifest("repo", "tag1", ispec.MediaTypeImageManifest, "digest",
[]byte("{}"), storeController, repoDB, log)
So(err, ShouldNotBeNil)
})
}
func TestUpdateErrors(t *testing.T) {
@ -160,8 +185,8 @@ func TestUpdateErrors(t *testing.T) {
repoDB := mocks.RepoDBMock{}
log := log.NewLogger("debug", "")
err := repoDBUpdate.SetMetadataFromInput("repo", "ref", "digest", "", []byte("BadManifestBlob"),
storeController, repoDB, log)
err := repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest",
[]byte("BadManifestBlob"), storeController, repoDB, log)
So(err, ShouldNotBeNil)
// reference is digest
@ -177,7 +202,7 @@ func TestUpdateErrors(t *testing.T) {
return []byte("{}"), nil
}
err = repoDBUpdate.SetMetadataFromInput("repo", string(godigest.FromString("reference")), "", "digest",
err = repodb.SetMetadataFromInput("repo", string(godigest.FromString("reference")), "", "digest",
manifestBlob, storeController, repoDB, log)
So(err, ShouldBeNil)
})

1
pkg/meta/repodb/version/version_test.go
View file

@ -119,6 +119,7 @@ func TestVersioningDynamoDB(t *testing.T) {
Region: region,
RepoMetaTablename: "RepoMetadataTable",
ManifestDataTablename: "ManifestDataTable",
IndexDataTablename: "IndexDataTable",
VersionTablename: "Version",
})
So(err, ShouldBeNil)

345
pkg/test/common.go
View file

@ -45,6 +45,7 @@ import (
"oras.land/oras-go/v2/registry/remote"
"oras.land/oras-go/v2/registry/remote/auth"
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/storage"
)
@ -85,13 +86,47 @@ var (
ErrAlreadyExists = errors.New("already exists")
ErrKeyNotFound = errors.New("key not found")
ErrSignatureVerification = errors.New("signature verification failed")
ErrPutIndex = errors.New("can't put index")
)
type Image struct {
Manifest ispec.Manifest
Config ispec.Image
Layers [][]byte
Tag string
Manifest ispec.Manifest
Config ispec.Image
Layers [][]byte
Reference string
}
func (img Image) Digest() (godigest.Digest, error) {
blob, err := json.Marshal(img.Manifest)
if err != nil {
return "", err
}
return godigest.FromBytes(blob), nil
}
type MultiarchImage struct {
Index ispec.Index
Images []Image
Reference string
}
func (mi *MultiarchImage) Digest() (godigest.Digest, error) {
indexBlob, err := json.Marshal(mi.Index)
if err != nil {
return "", err
}
return godigest.FromBytes(indexBlob), nil
}
func (mi *MultiarchImage) IndexData() (repodb.IndexData, error) {
indexBlob, err := json.Marshal(mi.Index)
if err != nil {
return repodb.IndexData{}, err
}
return repodb.IndexData{IndexBlob: indexBlob}, nil
}
func GetFreePort() string {
@ -298,7 +333,7 @@ func WriteImageToFileSystem(image Image, repoName string, storeController storag
return err
}
_, err = store.PutImageManifest(repoName, image.Tag, ispec.MediaTypeImageManifest, manifestBlob)
_, err = store.PutImageManifest(repoName, image.Reference, ispec.MediaTypeImageManifest, manifestBlob)
if err != nil {
return err
}
@ -306,6 +341,34 @@ func WriteImageToFileSystem(image Image, repoName string, storeController storag
return nil
}
func WriteMultiArchImageToFileSystem(multiarchImage MultiarchImage, repoName string,
storeController storage.StoreController,
) error {
store := storeController.GetImageStore(repoName)
err := store.InitRepo(repoName)
if err != nil {
return err
}
for _, image := range multiarchImage.Images {
err := WriteImageToFileSystem(image, repoName, storeController)
if err != nil {
return err
}
}
indexBlob, err := json.Marshal(multiarchImage.Index)
if err != nil {
return err
}
_, err = store.PutImageManifest(repoName, multiarchImage.Reference, ispec.MediaTypeImageIndex,
indexBlob)
return err
}
func WaitTillServerReady(url string) {
for {
_, err := resty.R().Get(url)
@ -417,7 +480,7 @@ func GetOciLayoutDigests(imagePath string) (godigest.Digest, godigest.Digest, go
oci, err := umoci.OpenLayout(imagePath)
if err != nil {
panic(err)
panic(fmt.Errorf("error opening layout at '%s' : %w", imagePath, err))
}
defer oci.Close()
@ -560,7 +623,7 @@ func GetRandomImageComponents(layerSize int) (ispec.Image, [][]byte, ispec.Manif
return config, layers, manifest, nil
}
func GetImageWithConfig(conf ispec.Image) (ispec.Image, [][]byte, ispec.Manifest, error) {
func GetImageComponentsWithConfig(conf ispec.Image) (ispec.Image, [][]byte, ispec.Manifest, error) {
configBlob, err := json.Marshal(conf)
if err = Error(err); err != nil {
return ispec.Image{}, [][]byte{}, ispec.Manifest{}, err
@ -603,6 +666,68 @@ func GetImageWithConfig(conf ispec.Image) (ispec.Image, [][]byte, ispec.Manifest
return conf, layers, manifest, nil
}
func GetImageWithConfig(conf ispec.Image) (Image, error) {
config, layers, manifest, err := GetImageComponentsWithConfig(conf)
if err != nil {
return Image{}, err
}
blob, err := json.Marshal(manifest)
if err != nil {
return Image{}, err
}
return Image{
Manifest: manifest,
Config: config,
Layers: layers,
Reference: godigest.FromBytes(blob).String(),
}, nil
}
func GetImageWithComponents(config ispec.Image, layers [][]byte) (Image, error) {
configBlob, err := json.Marshal(config)
if err != nil {
return Image{}, err
}
manifestLayers := make([]ispec.Descriptor, 0, len(layers))
for _, layer := range layers {
manifestLayers = append(manifestLayers, ispec.Descriptor{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: godigest.FromBytes(layer),
Size: int64(len(layer)),
})
}
const schemaVersion = 2
manifest := ispec.Manifest{
Versioned: specs.Versioned{
SchemaVersion: schemaVersion,
},
Config: ispec.Descriptor{
MediaType: "application/vnd.oci.image.config.v1+json",
Digest: godigest.FromBytes(configBlob),
Size: int64(len(configBlob)),
},
Layers: manifestLayers,
}
manifestBlob, err := json.Marshal(manifest)
if err != nil {
return Image{}, err
}
return Image{
Manifest: manifest,
Config: config,
Layers: layers,
Reference: godigest.FromBytes(manifestBlob).String(),
}, nil
}
func GetCosignSignatureTagForManifest(manifest ispec.Manifest) (string, error) {
manifestBlob, err := json.Marshal(manifest)
if err != nil {
@ -692,7 +817,11 @@ func UploadImage(img Image, baseURL, repo string) error {
resp, err = resty.R().
SetHeader("Content-type", "application/vnd.oci.image.manifest.v1+json").
SetBody(manifestBlob).
Put(baseURL + "/v2/" + repo + "/manifests/" + img.Tag)
Put(baseURL + "/v2/" + repo + "/manifests/" + img.Reference)
if ErrStatusCode(resp.StatusCode()) != http.StatusCreated {
return ErrPutBlob
}
if ErrStatusCode(resp.StatusCode()) != http.StatusCreated {
return ErrPutBlob
@ -756,10 +885,10 @@ func PushTestImage(repoName string, tag string, //nolint:unparam
) error {
err := UploadImage(
Image{
Manifest: manifest,
Config: config,
Layers: layers,
Tag: tag,
Manifest: manifest,
Config: config,
Layers: layers,
Reference: tag,
},
baseURL,
repoName,
@ -1332,7 +1461,7 @@ func UploadImageWithBasicAuth(img Image, baseURL, repo, user, password string) e
SetBasicAuth(user, password).
SetHeader("Content-type", "application/vnd.oci.image.manifest.v1+json").
SetBody(manifestBlob).
Put(baseURL + "/v2/" + repo + "/manifests/" + img.Tag)
Put(baseURL + "/v2/" + repo + "/manifests/" + img.Reference)
return err
}
@ -1364,8 +1493,10 @@ func SignImageUsingCosign(repoTag, port string) error {
imageURL := fmt.Sprintf("localhost:%s/%s", port, repoTag)
const timeoutPeriod = 5
// sign the image
return sign.SignCmd(&options.RootOptions{Verbose: true, Timeout: 1 * time.Minute},
return sign.SignCmd(&options.RootOptions{Verbose: true, Timeout: timeoutPeriod * time.Minute},
options.KeyOpts{KeyRef: path.Join(tdir, "cosign.key"), PassFunc: generate.GetPass},
options.RegistryOptions{AllowInsecure: true},
map[string]interface{}{"tag": "1.0"},
@ -1408,3 +1539,189 @@ func SignImageUsingNotary(repoTag, port string) error {
return err
}
func GetRandomMultiarchImageComponents() (ispec.Index, []Image, error) {
const layerSize = 100
randomLayer1 := make([]byte, layerSize)
_, err := rand.Read(randomLayer1)
if err != nil {
return ispec.Index{}, []Image{}, err
}
image1, err := GetImageWithComponents(
ispec.Image{
Platform: ispec.Platform{
OS: "linux",
Architecture: "amd64",
},
},
[][]byte{
randomLayer1,
})
if err != nil {
return ispec.Index{}, []Image{}, err
}
image1.Reference = getManifestDigest(image1.Manifest).String()
randomLayer2 := make([]byte, layerSize)
_, err = rand.Read(randomLayer2)
if err != nil {
return ispec.Index{}, []Image{}, err
}
image2, err := GetImageWithComponents(
ispec.Image{
Platform: ispec.Platform{
OS: "linux",
Architecture: "386",
},
},
[][]byte{
randomLayer2,
})
if err != nil {
return ispec.Index{}, []Image{}, err
}
image2.Reference = getManifestDigest(image2.Manifest).String()
randomLayer3 := make([]byte, layerSize)
_, err = rand.Read(randomLayer3)
if err != nil {
return ispec.Index{}, []Image{}, err
}
image3, err := GetImageWithComponents(
ispec.Image{
Platform: ispec.Platform{
OS: "windows",
Architecture: "amd64",
},
},
[][]byte{
randomLayer3,
})
if err != nil {
return ispec.Index{}, []Image{}, err
}
image3.Reference = getManifestDigest(image3.Manifest).String()
index := ispec.Index{
MediaType: ispec.MediaTypeImageIndex,
Manifests: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageManifest,
Digest: getManifestDigest(image1.Manifest),
Size: getManifestSize(image1.Manifest),
},
{
MediaType: ispec.MediaTypeImageManifest,
Digest: getManifestDigest(image2.Manifest),
Size: getManifestSize(image2.Manifest),
},
{
MediaType: ispec.MediaTypeImageManifest,
Digest: getManifestDigest(image3.Manifest),
Size: getManifestSize(image3.Manifest),
},
},
}
return index, []Image{image1, image2, image3}, nil
}
func GetRandomMultiarchImage(reference string) (MultiarchImage, error) {
index, images, err := GetRandomMultiarchImageComponents()
if err != nil {
return MultiarchImage{}, err
}
return MultiarchImage{
Index: index, Images: images, Reference: reference,
}, err
}
func GetMultiarchImageForImages(reference string, images []Image) MultiarchImage {
var index ispec.Index
for i, image := range images {
index.Manifests = append(index.Manifests, ispec.Descriptor{
MediaType: ispec.MediaTypeImageManifest,
Digest: getManifestDigest(image.Manifest),
Size: getManifestSize(image.Manifest),
})
// update the reference with the digest of the manifest
images[i].Reference = getManifestDigest(image.Manifest).String()
}
return MultiarchImage{Index: index, Images: images, Reference: reference}
}
func getManifestSize(manifest ispec.Manifest) int64 {
manifestBlob, err := json.Marshal(manifest)
if err != nil {
return 0
}
return int64(len(manifestBlob))
}
func getManifestDigest(manifest ispec.Manifest) godigest.Digest {
manifestBlob, err := json.Marshal(manifest)
if err != nil {
return ""
}
return godigest.FromBytes(manifestBlob)
}
func UploadMultiarchImage(multiImage MultiarchImage, baseURL string, repo string) error {
for _, image := range multiImage.Images {
err := UploadImage(image, baseURL, repo)
if err != nil {
return err
}
}
// put manifest
indexBlob, err := json.Marshal(multiImage.Index)
if err = Error(err); err != nil {
return err
}
resp, err := resty.R().
SetHeader("Content-type", ispec.MediaTypeImageIndex).
SetBody(indexBlob).
Put(baseURL + "/v2/" + repo + "/manifests/" + multiImage.Reference)
if resp.StatusCode() != http.StatusCreated {
return ErrPutIndex
}
return err
}
func GetIndexBlobWithManifests(manifestDigests []godigest.Digest) ([]byte, error) {
manifests := make([]ispec.Descriptor, 0, len(manifestDigests))
for _, manifestDigest := range manifestDigests {
manifests = append(manifests, ispec.Descriptor{
Digest: manifestDigest,
MediaType: ispec.MediaTypeImageManifest,
})
}
indexContent := ispec.Index{
MediaType: ispec.MediaTypeImageIndex,
Manifests: manifests,
}
return json.Marshal(indexContent)
}

8
pkg/test/common_test.go
View file

@ -1061,10 +1061,10 @@ func TestVerifyWithNotation(t *testing.T) {
err = test.UploadImage(
test.Image{
Config: cfg,
Layers: layers,
Manifest: manifest,
Tag: tag,
Config: cfg,
Layers: layers,
Manifest: manifest,
Reference: tag,
}, baseURL, repoName)
So(err, ShouldBeNil)

25
pkg/test/mocks/cve_mock.go
View file

@ -9,10 +9,12 @@ import (
type CveInfoMock struct {
GetImageListForCVEFn func(repo, cveID string) ([]common.TagInfo, error)
GetImageListWithCVEFixedFn func(repo, cveID string) ([]common.TagInfo, error)
GetCVEListForImageFn func(image string, pageInput cveinfo.PageInput) ([]cvemodel.CVE, cveinfo.PageInfo, error)
GetCVESummaryForImageFn func(image string) (cveinfo.ImageCVESummary, error)
CompareSeveritiesFn func(severity1, severity2 string) int
UpdateDBFn func() error
GetCVEListForImageFn func(repo string, reference string, pageInput cveinfo.PageInput,
) ([]cvemodel.CVE, cveinfo.PageInfo, error)
GetCVESummaryForImageFn func(repo string, reference string,
) (cveinfo.ImageCVESummary, error)
CompareSeveritiesFn func(severity1, severity2 string) int
UpdateDBFn func() error
}
func (cveInfo CveInfoMock) GetImageListForCVE(repo, cveID string) ([]common.TagInfo, error) {
@ -31,21 +33,22 @@ func (cveInfo CveInfoMock) GetImageListWithCVEFixed(repo, cveID string) ([]commo
return []common.TagInfo{}, nil
}
func (cveInfo CveInfoMock) GetCVEListForImage(image string, pageInput cveinfo.PageInput) (
func (cveInfo CveInfoMock) GetCVEListForImage(repo string, reference string, pageInput cveinfo.PageInput) (
[]cvemodel.CVE,
cveinfo.PageInfo,
error,
) {
if cveInfo.GetCVEListForImageFn != nil {
return cveInfo.GetCVEListForImageFn(image, pageInput)
return cveInfo.GetCVEListForImageFn(repo, reference, pageInput)
}
return []cvemodel.CVE{}, cveinfo.PageInfo{}, nil
}
func (cveInfo CveInfoMock) GetCVESummaryForImage(image string) (cveinfo.ImageCVESummary, error) {
func (cveInfo CveInfoMock) GetCVESummaryForImage(repo string, reference string,
) (cveinfo.ImageCVESummary, error) {
if cveInfo.GetCVESummaryForImageFn != nil {
return cveInfo.GetCVESummaryForImageFn(image)
return cveInfo.GetCVESummaryForImageFn(repo, reference)
}
return cveinfo.ImageCVESummary{}, nil
@ -68,15 +71,15 @@ func (cveInfo CveInfoMock) UpdateDB() error {
}
type CveScannerMock struct {
IsImageFormatScannableFn func(image string) (bool, error)
IsImageFormatScannableFn func(repo string, reference string) (bool, error)
ScanImageFn func(image string) (map[string]cvemodel.CVE, error)
CompareSeveritiesFn func(severity1, severity2 string) int
UpdateDBFn func() error
}
func (scanner CveScannerMock) IsImageFormatScannable(image string) (bool, error) {
func (scanner CveScannerMock) IsImageFormatScannable(repo string, reference string) (bool, error) {
if scanner.IsImageFormatScannableFn != nil {
return scanner.IsImageFormatScannableFn(image)
return scanner.IsImageFormatScannableFn(repo, reference)
}
return true, nil

45
pkg/test/mocks/repo_db_mock.go
View file

@ -36,6 +36,10 @@ type RepoDBMock struct {
SetManifestMetaFn func(repo string, manifestDigest godigest.Digest, mm repodb.ManifestMetadata) error
SetIndexDataFn func(digest godigest.Digest, indexData repodb.IndexData) error
GetIndexDataFn func(indexDigest godigest.Digest) (repodb.IndexData, error)
IncrementImageDownloadsFn func(repo string, reference string) error
AddManifestSignatureFn func(repo string, signedManifestDigest godigest.Digest, sm repodb.SignatureMetadata) error
@ -43,14 +47,14 @@ type RepoDBMock struct {
DeleteSignatureFn func(repo string, signedManifestDigest godigest.Digest, sm repodb.SignatureMetadata) error
SearchReposFn func(ctx context.Context, searchText string, filter repodb.Filter, requestedPage repodb.PageInput) (
[]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error)
[]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error)
SearchTagsFn func(ctx context.Context, searchText string, filter repodb.Filter, requestedPage repodb.PageInput) (
[]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error)
[]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error)
FilterTagsFn func(ctx context.Context, filter repodb.FilterFunc,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error)
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error)
SearchDigestsFn func(ctx context.Context, searchText string, requestedPage repodb.PageInput) (
[]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error)
@ -143,7 +147,7 @@ func (sdm RepoDBMock) GetMultipleRepoMeta(ctx context.Context, filter func(repoM
func (sdm RepoDBMock) GetManifestData(manifestDigest godigest.Digest) (repodb.ManifestData, error) {
if sdm.GetManifestDataFn != nil {
return sdm.GetManifestData(manifestDigest)
return sdm.GetManifestDataFn(manifestDigest)
}
return repodb.ManifestData{}, nil
@ -151,7 +155,7 @@ func (sdm RepoDBMock) GetManifestData(manifestDigest godigest.Digest) (repodb.Ma
func (sdm RepoDBMock) SetManifestData(manifestDigest godigest.Digest, md repodb.ManifestData) error {
if sdm.SetManifestDataFn != nil {
return sdm.SetManifestData(manifestDigest, md)
return sdm.SetManifestDataFn(manifestDigest, md)
}
return nil
@ -203,32 +207,35 @@ func (sdm RepoDBMock) DeleteSignature(repo string, signedManifestDigest godigest
func (sdm RepoDBMock) SearchRepos(ctx context.Context, searchText string, filter repodb.Filter,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error) {
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
if sdm.SearchReposFn != nil {
return sdm.SearchReposFn(ctx, searchText, filter, requestedPage)
}
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, repodb.PageInfo{}, nil
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{},
map[string]repodb.IndexData{}, repodb.PageInfo{}, nil
}
func (sdm RepoDBMock) SearchTags(ctx context.Context, searchText string, filter repodb.Filter,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error) {
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
if sdm.SearchTagsFn != nil {
return sdm.SearchTagsFn(ctx, searchText, filter, requestedPage)
}
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, repodb.PageInfo{}, nil
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{},
map[string]repodb.IndexData{}, repodb.PageInfo{}, nil
}
func (sdm RepoDBMock) FilterTags(ctx context.Context, filter repodb.FilterFunc,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, repodb.PageInfo, error) {
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
if sdm.FilterTagsFn != nil {
return sdm.FilterTagsFn(ctx, filter, requestedPage)
}
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, repodb.PageInfo{}, nil
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{},
map[string]repodb.IndexData{}, repodb.PageInfo{}, nil
}
func (sdm RepoDBMock) SearchDigests(ctx context.Context, searchText string, requestedPage repodb.PageInput,
@ -268,6 +275,22 @@ func (sdm RepoDBMock) SearchForDescendantImages(ctx context.Context, searchText
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, nil
}
func (sdm RepoDBMock) SetIndexData(digest godigest.Digest, indexData repodb.IndexData) error {
if sdm.SetIndexDataFn != nil {
return sdm.SetIndexDataFn(digest, indexData)
}
return nil
}
func (sdm RepoDBMock) GetIndexData(indexDigest godigest.Digest) (repodb.IndexData, error) {
if sdm.GetIndexDataFn != nil {
return sdm.GetIndexDataFn(indexDigest)
}
return repodb.IndexData{}, nil
}
func (sdm RepoDBMock) PatchDB() error {
if sdm.PatchDBFn != nil {
return sdm.PatchDBFn()

11
test/blackbox/annotations.bats
View file

@ -74,7 +74,8 @@ function teardown_file() {
[ "$status" -eq 0 ]
run podman push 127.0.0.1:8080/annotations:latest --tls-verify=false --format=oci
[ "$status" -eq 0 ]
run curl -X POST -H "Content-Type: application/json" --data '{ "query": "{ ImageList(repo: \"annotations\") { Results { RepoName Tag Digest ConfigDigest Size Layers {Size Digest } Vendor Licenses }}}"}' http://localhost:8080/v2/_zot/ext/search
run curl -X POST -H "Content-Type: application/json" --data '{ "query": "{ ImageList(repo: \"annotations\") { Results { RepoName Tag Manifests {Digest ConfigDigest Size Layers { Size Digest }} Vendor Licenses }}}"}' http://localhost:8080/v2/_zot/ext/search
[ "$status" -eq 0 ]
# [ $(echo "${lines[-1]}" | jq '.data.ImageList') ]
[ $(echo "${lines[-1]}" | jq '.data.ImageList.Results[0].RepoName') = '"annotations"' ]
@ -87,7 +88,7 @@ function teardown_file() {
[ "$status" -eq 0 ]
run stacker --oci-dir ${BATS_FILE_TMPDIR}/stackeroci --stacker-dir ${BATS_FILE_TMPDIR}/.stacker --roots-dir ${BATS_FILE_TMPDIR}/roots publish -f ${BATS_FILE_TMPDIR}/stacker.yaml --substitute IMAGE_NAME="ghcr.io/project-zot/golang" --substitute IMAGE_TAG="1.20" --substitute DESCRIPTION="mydesc" --substitute VENDOR="CentOs" --substitute LICENSES="GPLv2" --url docker://127.0.0.1:8080 --tag 1.20 --skip-tls
[ "$status" -eq 0 ]
run curl -X POST -H "Content-Type: application/json" --data '{ "query": "{ ImageList(repo: \"ghcr.io/project-zot/golang\") { Results { RepoName Tag Digest ConfigDigest Size Layers {Size Digest } Description Vendor Licenses }}}"}' http://localhost:8080/v2/_zot/ext/search
run curl -X POST -H "Content-Type: application/json" --data '{ "query": "{ ImageList(repo: \"ghcr.io/project-zot/golang\") { Results { RepoName Tag Manifests {Digest ConfigDigest Size Layers { Size Digest }} Vendor Licenses Description }}}"}' http://localhost:8080/v2/_zot/ext/search
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.data.ImageList.Results[0].RepoName') = '"ghcr.io/project-zot/golang"' ]
[ $(echo "${lines[-1]}" | jq '.data.ImageList.Results[0].Description') = '"mydesc"' ]
@ -96,10 +97,10 @@ function teardown_file() {
}
@test "sign/verify with cosign" {
run curl -X POST -H "Content-Type: application/json" --data '{ "query": "{ ImageList(repo: \"annotations\") { Results { RepoName Tag Digest ConfigDigest Size Layers {Size Digest } Vendor Licenses }}}"}' http://localhost:8080/v2/_zot/ext/search
run curl -X POST -H "Content-Type: application/json" --data '{ "query": "{ ImageList(repo: \"annotations\") { Results { RepoName Tag Manifests {Digest ConfigDigest Size Layers { Size Digest }} Vendor Licenses }}}"}' http://localhost:8080/v2/_zot/ext/search
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.data.ImageList.Results[0].RepoName') = '"annotations"' ]
local digest=$(echo "${lines[-1]}" | jq -r '.data.ImageList.Results[0].Digest')
local digest=$(echo "${lines[-1]}" | jq -r '.data.ImageList.Results[0].Manifests[0].Digest')
run cosign initialize
[ "$status" -eq 0 ]
@ -115,7 +116,7 @@ function teardown_file() {
}
@test "sign/verify with notation" {
run curl -X POST -H "Content-Type: application/json" --data '{ "query": "{ ImageList(repo: \"annotations\") { Results { RepoName Tag Digest ConfigDigest Size Layers {Size Digest } }}}"}' http://localhost:8080/v2/_zot/ext/search
run curl -X POST -H "Content-Type: application/json" --data '{ "query": "{ ImageList(repo: \"annotations\") { Results { RepoName Tag Manifests {Digest ConfigDigest Size Layers { Size Digest }} Vendor Licenses }}}"}' http://localhost:8080/v2/_zot/ext/search
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.data.ImageList.Results[0].RepoName') = '"annotations"' ]
[ "$status" -eq 0 ]

3
test/blackbox/cloud-only.bats
View file

@ -37,6 +37,7 @@ function setup() {
"cacheTablename": "BlobTable",
"repoMetaTablename": "RepoMetadataTable",
"manifestDataTablename": "ManifestDataTable",
"indexDataTablename": "IndexDataTable",
"versionTablename": "Version"
}
},
@ -66,8 +67,6 @@ function setup() {
EOF
awslocal s3 --region "us-east-2" mb s3://zot-storage
awslocal dynamodb --region "us-east-2" create-table --table-name "BlobTable" --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
awslocal dynamodb --region "us-east-2" create-table --table-name "RepoMetadataTable" --attribute-definitions AttributeName=RepoName,AttributeType=S --key-schema AttributeName=RepoName,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
awslocal dynamodb --region "us-east-2" create-table --table-name "ManifestDataTable" --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
zot_serve_strace ${zot_config_file}
wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
}