mirror of
https://github.com/project-zot/zot.git
synced 2024-12-16 21:56:37 -05:00
feat(repodb): update referrers api to use repodb (#1230)
Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
This commit is contained in:
parent
c731acf6de
commit
5d1f91a79f
37 changed files with 2011 additions and 201 deletions
7
.github/workflows/ci-cd.yml
vendored
7
.github/workflows/ci-cd.yml
vendored
|
@ -89,11 +89,10 @@ jobs:
|
|||
|
||||
echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container
|
||||
localstack wait -t 30 # to become ready before timing out
|
||||
echo "Startup complete"
|
||||
|
||||
echo "Startup complete"
|
||||
|
||||
aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}"
|
||||
aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
|
||||
aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name RepoMetadataTable --attribute-definitions AttributeName=RepoName,AttributeType=S --key-schema AttributeName=RepoName,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
|
||||
aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name ManifestDataTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: fake
|
||||
AWS_SECRET_ACCESS_KEY: fake
|
||||
|
|
3
.github/workflows/ecosystem-tools.yaml
vendored
3
.github/workflows/ecosystem-tools.yaml
vendored
|
@ -33,6 +33,9 @@ jobs:
|
|||
make bin/skopeo
|
||||
sudo cp bin/skopeo /usr/bin
|
||||
skopeo -v
|
||||
- name: Run referrers tests
|
||||
run: |
|
||||
make test-bats-referrers
|
||||
- name: Run push-pull tests
|
||||
run: |
|
||||
make test-push-pull
|
||||
|
|
5
Makefile
5
Makefile
|
@ -298,6 +298,11 @@ test-push-pull: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
|
|||
test-push-pull-verbose: binary check-skopeo $(BATS)
|
||||
$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull.bats
|
||||
|
||||
.PHONY: test-bats-referrers
|
||||
test-bats-referrers: EXTENSIONS=search
|
||||
test-bats-referrers: binary check-skopeo $(BATS) $(ORAS)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/referrers.bats
|
||||
|
||||
.PHONY: test-cloud-only
|
||||
test-cloud-only: binary check-skopeo $(BATS)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/cloud-only.bats
|
||||
|
|
|
@ -63,6 +63,7 @@ var (
|
|||
ErrManifestConflict = errors.New("manifest: multiple manifests found")
|
||||
ErrManifestMetaNotFound = errors.New("repodb: image metadata not found for given manifest reference")
|
||||
ErrManifestDataNotFound = errors.New("repodb: image data not found for given manifest digest")
|
||||
ErrArtifactDataNotFound = errors.New("repodb: artifact data not found for given digest")
|
||||
ErrIndexDataNotFount = errors.New("repodb: index data not found for given digest")
|
||||
ErrRepoMetaNotFound = errors.New("repodb: repo metadata not found for given repo name")
|
||||
ErrTagMetaNotFound = errors.New("repodb: tag metadata not found for given repo and tag names")
|
||||
|
|
|
@ -20,6 +20,7 @@
|
|||
"cacheTablename": "ZotBlobTable",
|
||||
"repoMetaTablename": "ZotRepoMetadataTable",
|
||||
"manifestDataTablename": "ZotManifestDataTable",
|
||||
"artifactDataTablename": "ZotArtifactDataTable",
|
||||
"versionTablename": "ZotVersion"
|
||||
}
|
||||
},
|
||||
|
|
|
@ -512,7 +512,7 @@ func (c *Controller) InitRepoDB(reloadCtx context.Context) error {
|
|||
return err
|
||||
}
|
||||
|
||||
err = repodb.SyncRepoDB(driver, c.StoreController, c.Log)
|
||||
err = repodb.ParseStorage(driver, c.StoreController, c.Log)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -554,6 +554,9 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) d
|
|||
indexDataTablename, ok := toStringIfOk(cacheDriverConfig, "indexdatatablename", log)
|
||||
allParametersOk = allParametersOk && ok
|
||||
|
||||
artifactDataTablename, ok := toStringIfOk(cacheDriverConfig, "artifactdatatablename", log)
|
||||
allParametersOk = allParametersOk && ok
|
||||
|
||||
versionTablename, ok := toStringIfOk(cacheDriverConfig, "versiontablename", log)
|
||||
allParametersOk = allParametersOk && ok
|
||||
|
||||
|
@ -567,6 +570,7 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) d
|
|||
RepoMetaTablename: repoMetaTablename,
|
||||
ManifestDataTablename: manifestDataTablename,
|
||||
IndexDataTablename: indexDataTablename,
|
||||
ArtifactDataTablename: artifactDataTablename,
|
||||
VersionTablename: versionTablename,
|
||||
}
|
||||
}
|
||||
|
|
|
@ -170,6 +170,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
|
|||
"cacheTablename": "BlobTable",
|
||||
"repoMetaTablename": "RepoMetadataTable",
|
||||
"manifestDataTablename": "ManifestDataTable",
|
||||
"artifactDataTablename": "ArtifactDataTable",
|
||||
"versionTablename": "Version",
|
||||
}
|
||||
|
||||
|
@ -184,6 +185,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
|
|||
"cacheTablename": "BlobTable",
|
||||
"repoMetaTablename": "RepoMetadataTable",
|
||||
"manifestDataTablename": "ManifestDataTable",
|
||||
"artifactDataTablename": "ArtifactDataTable",
|
||||
"versionTablename": "Version",
|
||||
}
|
||||
|
||||
|
@ -197,6 +199,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
|
|||
"cacheTablename": "BlobTable",
|
||||
"repoMetaTablename": "RepoMetadataTable",
|
||||
"manifestDataTablename": "ManifestDataTable",
|
||||
"artifactDataTablename": "ArtifactDataTable",
|
||||
"versionTablename": "Version",
|
||||
}
|
||||
|
||||
|
@ -229,6 +232,7 @@ func TestCreateRepoDBDriver(t *testing.T) {
|
|||
"cachetablename": "BlobTable",
|
||||
"repometatablename": "RepoMetadataTable",
|
||||
"manifestdatatablename": "ManifestDataTable",
|
||||
"artifactDataTablename": "ArtifactDataTable",
|
||||
}
|
||||
|
||||
testFunc := func() { _, _ = api.CreateRepoDBDriver(conf.Storage.StorageConfig, log) }
|
||||
|
@ -241,6 +245,7 @@ func TestCreateRepoDBDriver(t *testing.T) {
|
|||
"cachetablename": "",
|
||||
"repometatablename": "RepoMetadataTable",
|
||||
"manifestdatatablename": "ManifestDataTable",
|
||||
"artifactDataTablename": "ArtifactDataTable",
|
||||
"versiontablename": 1,
|
||||
}
|
||||
|
||||
|
@ -2574,6 +2579,7 @@ func TestAuthorizationWithBasicAuth(t *testing.T) {
|
|||
|
||||
manifestBlob := resp.Body()
|
||||
var manifest ispec.Manifest
|
||||
|
||||
err = json.Unmarshal(manifestBlob, &manifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
|
|
|
@ -1008,6 +1008,7 @@ func TestGetReferrersGQL(t *testing.T) {
|
|||
},
|
||||
Subject: subjectDescriptor,
|
||||
ArtifactType: artifactType,
|
||||
MediaType: ispec.MediaTypeArtifactManifest,
|
||||
Annotations: map[string]string{
|
||||
"com.artifact.format": "test",
|
||||
},
|
||||
|
@ -1017,14 +1018,15 @@ func TestGetReferrersGQL(t *testing.T) {
|
|||
So(err, ShouldBeNil)
|
||||
artifactManifestDigest := godigest.FromBytes(artifactManifestBlob)
|
||||
|
||||
err = UploadArtifact(baseURL, repo, artifact)
|
||||
err = UploadArtifactManifest(artifact, baseURL, repo)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
gqlQuery := `
|
||||
{Referrers(
|
||||
repo: "%s",
|
||||
digest: "%s",
|
||||
type: ""
|
||||
{
|
||||
Referrers(
|
||||
repo: "%s",
|
||||
digest: "%s",
|
||||
type: ""
|
||||
){
|
||||
ArtifactType,
|
||||
Digest,
|
||||
|
@ -5356,20 +5358,15 @@ func TestRepoDBWhenReadingImages(t *testing.T) {
|
|||
|
||||
func TestRepoDBWhenDeletingImages(t *testing.T) {
|
||||
Convey("Setting up zot repo with test images", t, func() {
|
||||
subpath := "/a"
|
||||
|
||||
dir := t.TempDir()
|
||||
subDir := t.TempDir()
|
||||
|
||||
subRootDir := path.Join(subDir, subpath)
|
||||
|
||||
port := GetFreePort()
|
||||
baseURL := GetBaseURL(port)
|
||||
|
||||
conf := config.New()
|
||||
conf.HTTP.Port = port
|
||||
conf.Storage.RootDirectory = dir
|
||||
conf.Storage.SubPaths = make(map[string]config.StorageConfig)
|
||||
conf.Storage.SubPaths[subpath] = config.StorageConfig{RootDirectory: subRootDir}
|
||||
conf.Storage.GC = false
|
||||
|
||||
defaultVal := true
|
||||
conf.Extensions = &extconf.ExtensionConfig{
|
||||
Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}},
|
||||
|
@ -5384,46 +5381,32 @@ func TestRepoDBWhenDeletingImages(t *testing.T) {
|
|||
defer ctlrManager.StopServer()
|
||||
|
||||
// push test images to repo 1 image 1
|
||||
config1, layers1, manifest1, err := GetImageComponents(100)
|
||||
image1, err := GetRandomImage("1.0.1")
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
layersSize1 := 0
|
||||
for _, l := range layers1 {
|
||||
layersSize1 += len(l)
|
||||
}
|
||||
|
||||
err = UploadImage(
|
||||
Image{
|
||||
Manifest: manifest1,
|
||||
Config: config1,
|
||||
Layers: layers1,
|
||||
Reference: "1.0.1",
|
||||
},
|
||||
image1,
|
||||
baseURL,
|
||||
"repo1",
|
||||
)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
// push test images to repo 1 image 2
|
||||
config2, layers2, manifest2, err := GetImageComponents(200)
|
||||
So(err, ShouldBeNil)
|
||||
createdTime2 := time.Date(2009, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
config2.History = append(config2.History, ispec.History{Created: &createdTime2})
|
||||
manifest2, err = updateManifestConfig(manifest2, config2)
|
||||
image2, err := GetImageWithConfig(ispec.Image{
|
||||
Created: &createdTime2,
|
||||
History: []ispec.History{
|
||||
{
|
||||
Created: &createdTime2,
|
||||
},
|
||||
},
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
layersSize2 := 0
|
||||
for _, l := range layers2 {
|
||||
layersSize2 += len(l)
|
||||
}
|
||||
image2.Reference = "1.0.2"
|
||||
|
||||
err = UploadImage(
|
||||
Image{
|
||||
Manifest: manifest2,
|
||||
Config: config2,
|
||||
Layers: layers2,
|
||||
Reference: "1.0.2",
|
||||
},
|
||||
image2,
|
||||
baseURL,
|
||||
"repo1",
|
||||
)
|
||||
|
@ -5439,22 +5422,6 @@ func TestRepoDBWhenDeletingImages(t *testing.T) {
|
|||
LastUpdated Size
|
||||
}
|
||||
}
|
||||
Repos {
|
||||
Name LastUpdated Size
|
||||
Platforms { Os Arch }
|
||||
Vendors Score
|
||||
NewestImage {
|
||||
RepoName Tag LastUpdated Size IsSigned
|
||||
Manifests{
|
||||
Platform { Os Arch }
|
||||
LastUpdated Size
|
||||
}
|
||||
}
|
||||
}
|
||||
Layers {
|
||||
Digest
|
||||
Size
|
||||
}
|
||||
}
|
||||
}`
|
||||
|
||||
|
@ -5627,7 +5594,7 @@ func TestRepoDBWhenDeletingImages(t *testing.T) {
|
|||
|
||||
So(sigManifestContent, ShouldNotBeZeroValue)
|
||||
// check notation signature
|
||||
manifest1Blob, err := json.Marshal(manifest1)
|
||||
manifest1Blob, err := json.Marshal(image1.Manifest)
|
||||
So(err, ShouldBeNil)
|
||||
manifest1Digest := godigest.FromBytes(manifest1Blob)
|
||||
So(sigManifestContent.Subject, ShouldNotBeNil)
|
||||
|
@ -5653,6 +5620,65 @@ func TestRepoDBWhenDeletingImages(t *testing.T) {
|
|||
So(responseStruct.GlobalSearchResult.GlobalSearch.Images[0].IsSigned, ShouldBeFalse)
|
||||
})
|
||||
|
||||
Convey("Delete a referrer", func() {
|
||||
referredImageDigest, err := image1.Digest()
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
referrerImage, err := GetImageWithSubject(referredImageDigest, ispec.MediaTypeImageManifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = UploadImage(
|
||||
referrerImage,
|
||||
baseURL,
|
||||
"repo1",
|
||||
)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
// ------- check referrers for this image
|
||||
|
||||
query := fmt.Sprintf(`
|
||||
{
|
||||
Referrers(repo:"repo1", digest:"%s"){
|
||||
MediaType
|
||||
Digest
|
||||
}
|
||||
}`, referredImageDigest.String())
|
||||
|
||||
resp, err = resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query))
|
||||
So(resp, ShouldNotBeNil)
|
||||
So(err, ShouldBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, 200)
|
||||
|
||||
responseStruct := &ReferrersResp{}
|
||||
|
||||
err = json.Unmarshal(resp.Body(), responseStruct)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(len(responseStruct.ReferrersResult.Referrers), ShouldEqual, 1)
|
||||
So(responseStruct.ReferrersResult.Referrers[0].Digest, ShouldResemble, referrerImage.Reference)
|
||||
|
||||
statusCode, err := DeleteImage("repo1", referrerImage.Reference, "badURL")
|
||||
So(err, ShouldNotBeNil)
|
||||
So(statusCode, ShouldEqual, -1)
|
||||
|
||||
// ------- Delete the referrer and see if it disappears from repoDB also
|
||||
statusCode, err = DeleteImage("repo1", referrerImage.Reference, baseURL)
|
||||
So(err, ShouldBeNil)
|
||||
So(statusCode, ShouldEqual, http.StatusAccepted)
|
||||
|
||||
resp, err = resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query))
|
||||
So(resp, ShouldNotBeNil)
|
||||
So(err, ShouldBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, 200)
|
||||
|
||||
responseStruct = &ReferrersResp{}
|
||||
|
||||
err = json.Unmarshal(resp.Body(), responseStruct)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(len(responseStruct.ReferrersResult.Referrers), ShouldEqual, 0)
|
||||
})
|
||||
|
||||
Convey("Deleting causes errors", func() {
|
||||
Convey("error while backing up the manifest", func() {
|
||||
ctlr.StoreController.DefaultStore = mocks.MockedImageStore{
|
||||
|
|
|
@ -627,6 +627,22 @@ func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta repodb.RepoMetadata
|
|||
return summary, imageSummaries
|
||||
}
|
||||
|
||||
func StringMap2Annotations(strMap map[string]string) []*gql_generated.Annotation {
|
||||
annotations := make([]*gql_generated.Annotation, 0, len(strMap))
|
||||
|
||||
for key, value := range strMap {
|
||||
key := key
|
||||
value := value
|
||||
|
||||
annotations = append(annotations, &gql_generated.Annotation{
|
||||
Key: &key,
|
||||
Value: &value,
|
||||
})
|
||||
}
|
||||
|
||||
return annotations
|
||||
}
|
||||
|
||||
func GetPreloads(ctx context.Context) map[string]bool {
|
||||
if !graphql.HasOperationContext(ctx) {
|
||||
return map[string]bool{}
|
||||
|
|
|
@ -317,7 +317,7 @@ func TestImageFormat(t *testing.T) {
|
|||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = repodb.SyncRepoDB(repoDB, storeController, log)
|
||||
err = repodb.ParseStorage(repoDB, storeController, log)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "", log)
|
||||
|
|
|
@ -88,7 +88,7 @@ func TestMultipleStoragePath(t *testing.T) {
|
|||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = repodb.SyncRepoDB(repoDB, storeController, log)
|
||||
err = repodb.ParseStorage(repoDB, storeController, log)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", log)
|
||||
|
@ -178,7 +178,7 @@ func TestTrivyLibraryErrors(t *testing.T) {
|
|||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = repodb.SyncRepoDB(repoDB, storeController, log)
|
||||
err = repodb.ParseStorage(repoDB, storeController, log)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", log)
|
||||
|
@ -424,7 +424,7 @@ func TestDefaultTrivyDBUrl(t *testing.T) {
|
|||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = repodb.SyncRepoDB(repoDB, storeController, log)
|
||||
err = repodb.ParseStorage(repoDB, storeController, log)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
// Use empty string for DB repository, the default url would be used internally
|
||||
|
|
|
@ -1193,41 +1193,33 @@ func getImageList(ctx context.Context, repo string, repoDB repodb.RepoDB, cveInf
|
|||
}, nil
|
||||
}
|
||||
|
||||
func getReferrers(store storage.ImageStore, repoName string, digest string, artifactTypes []string, log log.Logger) (
|
||||
[]*gql_generated.Referrer, error,
|
||||
) {
|
||||
results := make([]*gql_generated.Referrer, 0)
|
||||
func getReferrers(repoDB repodb.RepoDB, repo string, referredDigest string, artifactTypes []string,
|
||||
log log.Logger,
|
||||
) ([]*gql_generated.Referrer, error) {
|
||||
refDigest := godigest.Digest(referredDigest)
|
||||
if err := refDigest.Validate(); err != nil {
|
||||
log.Error().Err(err).Msgf("graphql: bad digest string from request '%s'", referredDigest)
|
||||
|
||||
index, err := store.GetReferrers(repoName, godigest.Digest(digest), artifactTypes)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("error extracting referrers list")
|
||||
|
||||
return results, err
|
||||
return []*gql_generated.Referrer{}, errors.Wrapf(err, "graphql: bad digest string from request '%s'",
|
||||
referredDigest)
|
||||
}
|
||||
|
||||
for _, manifest := range index.Manifests {
|
||||
size := int(manifest.Size)
|
||||
digest := manifest.Digest.String()
|
||||
annotations := make([]*gql_generated.Annotation, 0)
|
||||
artifactType := manifest.ArtifactType
|
||||
mediaType := manifest.MediaType
|
||||
referrers, err := repoDB.GetFilteredReferrersInfo(repo, refDigest, artifactTypes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for k, v := range manifest.Annotations {
|
||||
key := k
|
||||
value := v
|
||||
results := make([]*gql_generated.Referrer, 0, len(referrers))
|
||||
|
||||
annotations = append(annotations, &gql_generated.Annotation{
|
||||
Key: &key,
|
||||
Value: &value,
|
||||
})
|
||||
}
|
||||
for _, referrer := range referrers {
|
||||
referrer := referrer
|
||||
|
||||
results = append(results, &gql_generated.Referrer{
|
||||
MediaType: &mediaType,
|
||||
ArtifactType: &artifactType,
|
||||
Digest: &digest,
|
||||
Size: &size,
|
||||
Annotations: annotations,
|
||||
MediaType: &referrer.MediaType,
|
||||
ArtifactType: &referrer.ArtifactType,
|
||||
Digest: &referrer.Digest,
|
||||
Size: &referrer.Size,
|
||||
Annotations: convert.StringMap2Annotations(referrer.Annotations),
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
@ -1362,15 +1362,25 @@ func TestImageList(t *testing.T) {
|
|||
|
||||
func TestGetReferrers(t *testing.T) {
|
||||
Convey("getReferrers", t, func() {
|
||||
referredDigest := godigest.FromString("t").String()
|
||||
|
||||
Convey("referredDigest is empty", func() {
|
||||
testLogger := log.NewLogger("debug", "")
|
||||
|
||||
_, err := getReferrers(mocks.RepoDBMock{}, "test", "", nil, testLogger)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("GetReferrers returns error", func() {
|
||||
testLogger := log.NewLogger("debug", "")
|
||||
mockedStore := mocks.MockedImageStore{
|
||||
GetReferrersFn: func(repo string, digest godigest.Digest, artifactType []string) (ispec.Index, error) {
|
||||
return ispec.Index{}, ErrTestError
|
||||
mockedStore := mocks.RepoDBMock{
|
||||
GetFilteredReferrersInfoFn: func(repo string, referredDigest godigest.Digest, artifactTypes []string,
|
||||
) ([]repodb.ReferrerInfo, error) {
|
||||
return nil, ErrTestError
|
||||
},
|
||||
}
|
||||
|
||||
_, err := getReferrers(mockedStore, "test", "", nil, testLogger)
|
||||
_, err := getReferrers(mockedStore, "test", referredDigest, nil, testLogger)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
|
@ -1385,17 +1395,22 @@ func TestGetReferrers(t *testing.T) {
|
|||
"key": "value",
|
||||
},
|
||||
}
|
||||
mockedStore := mocks.MockedImageStore{
|
||||
GetReferrersFn: func(repo string, digest godigest.Digest, artifactTypes []string) (ispec.Index, error) {
|
||||
return ispec.Index{
|
||||
Manifests: []ispec.Descriptor{
|
||||
referrerDescriptor,
|
||||
mockedStore := mocks.RepoDBMock{
|
||||
GetFilteredReferrersInfoFn: func(repo string, referredDigest godigest.Digest, artifactTypes []string,
|
||||
) ([]repodb.ReferrerInfo, error) {
|
||||
return []repodb.ReferrerInfo{
|
||||
{
|
||||
Digest: referrerDescriptor.Digest.String(),
|
||||
MediaType: referrerDescriptor.MediaType,
|
||||
ArtifactType: referrerDescriptor.ArtifactType,
|
||||
Size: int(referrerDescriptor.Size),
|
||||
Annotations: referrerDescriptor.Annotations,
|
||||
},
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
|
||||
referrers, err := getReferrers(mockedStore, "test", "", nil, testLogger)
|
||||
referrers, err := getReferrers(mockedStore, "test", referredDigest, nil, testLogger)
|
||||
So(err, ShouldBeNil)
|
||||
So(*referrers[0].ArtifactType, ShouldEqual, referrerDescriptor.ArtifactType)
|
||||
So(*referrers[0].MediaType, ShouldEqual, referrerDescriptor.MediaType)
|
||||
|
|
|
@ -130,9 +130,7 @@ func (r *queryResolver) Image(ctx context.Context, image string) (*gql_generated
|
|||
|
||||
// Referrers is the resolver for the Referrers field.
|
||||
func (r *queryResolver) Referrers(ctx context.Context, repo string, digest string, typeArg []string) ([]*gql_generated.Referrer, error) {
|
||||
store := r.storeController.GetImageStore(repo)
|
||||
|
||||
referrers, err := getReferrers(store, repo, digest, typeArg, r.log)
|
||||
referrers, err := getReferrers(r.repoDB, repo, digest, typeArg, r.log)
|
||||
if err != nil {
|
||||
r.log.Error().Err(err).Msg("unable to get referrers from default store")
|
||||
|
||||
|
|
|
@ -61,6 +61,11 @@ func NewBoltDBWrapper(params DBParameters) (*DBWrapper, error) {
|
|||
return err
|
||||
}
|
||||
|
||||
_, err = transaction.CreateBucketIfNotExists([]byte(repodb.ArtifactDataBucket))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = transaction.CreateBucketIfNotExists([]byte(repodb.RepoMetadataBucket))
|
||||
if err != nil {
|
||||
return err
|
||||
|
@ -133,6 +138,7 @@ func (bdw *DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Diges
|
|||
Tags: map[string]repodb.Descriptor{},
|
||||
Statistics: map[string]repodb.DescriptorStatistics{},
|
||||
Signatures: map[string]repodb.ManifestSignatures{},
|
||||
Referrers: map[string][]repodb.Descriptor{},
|
||||
}
|
||||
|
||||
repoMetaBlob := repoBuck.Get([]byte(repo))
|
||||
|
@ -259,6 +265,290 @@ func (bdw *DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexDat
|
|||
return indexMetadata, err
|
||||
}
|
||||
|
||||
func (bdw DBWrapper) SetArtifactData(artifactDigest godigest.Digest, artifactData repodb.ArtifactData) error {
|
||||
err := bdw.DB.Update(func(tx *bolt.Tx) error {
|
||||
buck := tx.Bucket([]byte(repodb.ArtifactDataBucket))
|
||||
|
||||
imBlob, err := json.Marshal(artifactData)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "repodb: error while calculating blob for artifact with digest %s", artifactDigest)
|
||||
}
|
||||
|
||||
err = buck.Put([]byte(artifactDigest), imBlob)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "repodb: error while setting artifact blob for digest %s", artifactDigest)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (bdw DBWrapper) GetArtifactData(artifactDigest godigest.Digest) (repodb.ArtifactData, error) {
|
||||
var artifactData repodb.ArtifactData
|
||||
|
||||
err := bdw.DB.View(func(tx *bolt.Tx) error {
|
||||
buck := tx.Bucket([]byte(repodb.ArtifactDataBucket))
|
||||
|
||||
blob := buck.Get([]byte(artifactDigest))
|
||||
|
||||
if len(blob) == 0 {
|
||||
return zerr.ErrArtifactDataNotFound
|
||||
}
|
||||
|
||||
err := json.Unmarshal(blob, &artifactData)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "repodb: error while unmashaling artifact data for digest %s", artifactDigest)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return artifactData, err
|
||||
}
|
||||
|
||||
func (bdw DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error {
|
||||
err := bdw.DB.Update(func(tx *bolt.Tx) error {
|
||||
buck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
|
||||
|
||||
repoMetaBlob := buck.Get([]byte(repo))
|
||||
|
||||
// object not found
|
||||
if len(repoMetaBlob) == 0 {
|
||||
var err error
|
||||
|
||||
// create a new object
|
||||
repoMeta := repodb.RepoMetadata{
|
||||
Name: repo,
|
||||
Tags: map[string]repodb.Descriptor{},
|
||||
Statistics: map[string]repodb.DescriptorStatistics{
|
||||
referredDigest.String(): {},
|
||||
},
|
||||
Signatures: map[string]repodb.ManifestSignatures{
|
||||
referredDigest.String(): {},
|
||||
},
|
||||
Referrers: map[string][]repodb.Descriptor{
|
||||
referredDigest.String(): {
|
||||
{
|
||||
Digest: referrer.Digest,
|
||||
MediaType: referrer.MediaType,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
repoMetaBlob, err = json.Marshal(repoMeta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return buck.Put([]byte(repo), repoMetaBlob)
|
||||
}
|
||||
var repoMeta repodb.RepoMetadata
|
||||
|
||||
err := json.Unmarshal(repoMetaBlob, &repoMeta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
refferers := repoMeta.Referrers[referredDigest.String()]
|
||||
|
||||
for i := range refferers {
|
||||
if refferers[i].Digest == referrer.Digest {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
refferers = append(refferers, repodb.Descriptor{
|
||||
Digest: referrer.Digest,
|
||||
MediaType: referrer.MediaType,
|
||||
})
|
||||
|
||||
repoMeta.Referrers[referredDigest.String()] = refferers
|
||||
|
||||
repoMetaBlob, err = json.Marshal(repoMeta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return buck.Put([]byte(repo), repoMetaBlob)
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (bdw DBWrapper) DeleteReferrer(repo string, referredDigest godigest.Digest,
|
||||
referrerDigest godigest.Digest,
|
||||
) error {
|
||||
return bdw.DB.Update(func(tx *bolt.Tx) error {
|
||||
buck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
|
||||
|
||||
repoMetaBlob := buck.Get([]byte(repo))
|
||||
|
||||
if len(repoMetaBlob) == 0 {
|
||||
return zerr.ErrRepoMetaNotFound
|
||||
}
|
||||
|
||||
var repoMeta repodb.RepoMetadata
|
||||
|
||||
err := json.Unmarshal(repoMetaBlob, &repoMeta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
referrers := repoMeta.Referrers[referredDigest.String()]
|
||||
|
||||
for i := range referrers {
|
||||
if referrers[i].Digest == referrerDigest.String() {
|
||||
referrers = append(referrers[:i], referrers[i+1:]...)
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
repoMeta.Referrers[referredDigest.String()] = referrers
|
||||
|
||||
repoMetaBlob, err = json.Marshal(repoMeta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return buck.Put([]byte(repo), repoMetaBlob)
|
||||
})
|
||||
}
|
||||
|
||||
func (bdw DBWrapper) GetReferrers(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error) {
|
||||
var referrers []repodb.Descriptor
|
||||
|
||||
err := bdw.DB.View(func(tx *bolt.Tx) error {
|
||||
buck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
|
||||
|
||||
repoMetaBlob := buck.Get([]byte(repo))
|
||||
if len(repoMetaBlob) == 0 {
|
||||
return zerr.ErrRepoMetaNotFound
|
||||
}
|
||||
|
||||
var repoMeta repodb.RepoMetadata
|
||||
|
||||
err := json.Unmarshal(repoMetaBlob, &repoMeta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
referrers = repoMeta.Referrers[referredDigest.String()]
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return referrers, err
|
||||
}
|
||||
|
||||
func (bdw DBWrapper) GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest,
|
||||
artifactTypes []string,
|
||||
) ([]repodb.ReferrerInfo, error) {
|
||||
referrersDescriptors, err := bdw.GetReferrers(repo, referredDigest)
|
||||
if err != nil {
|
||||
bdw.Log.Error().Msgf("repodb: failed to get referrers for '%s@%s'", repo, referredDigest.String())
|
||||
|
||||
return nil, err
|
||||
}
|
||||
|
||||
referrersInfo := []repodb.ReferrerInfo{}
|
||||
|
||||
err = bdw.DB.View(func(tx *bolt.Tx) error {
|
||||
artifactBuck := tx.Bucket([]byte(repodb.ArtifactDataBucket))
|
||||
manifestBuck := tx.Bucket([]byte(repodb.ManifestDataBucket))
|
||||
|
||||
for _, descriptor := range referrersDescriptors {
|
||||
referrerInfo := repodb.ReferrerInfo{}
|
||||
|
||||
switch descriptor.MediaType {
|
||||
case ispec.MediaTypeImageManifest:
|
||||
manifestDataBlob := manifestBuck.Get([]byte(descriptor.Digest))
|
||||
|
||||
if len(manifestDataBlob) == 0 {
|
||||
bdw.Log.Error().Msgf("repodb: manifest data not found for digest %s", descriptor.Digest)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
var manifestData repodb.ManifestData
|
||||
|
||||
err = json.Unmarshal(manifestDataBlob, &manifestData)
|
||||
if err != nil {
|
||||
bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal manifest data for digest %s",
|
||||
descriptor.Digest)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
var manifestContent ispec.Manifest
|
||||
|
||||
err := json.Unmarshal(manifestData.ManifestBlob, &manifestContent)
|
||||
if err != nil {
|
||||
bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal manifest for digest %s",
|
||||
descriptor.Digest)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
referrerInfo = repodb.ReferrerInfo{
|
||||
Digest: descriptor.Digest,
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
ArtifactType: manifestContent.Config.MediaType,
|
||||
Size: len(manifestData.ManifestBlob),
|
||||
Annotations: manifestContent.Annotations,
|
||||
}
|
||||
case ispec.MediaTypeArtifactManifest:
|
||||
artifactDataBlob := artifactBuck.Get([]byte(descriptor.Digest))
|
||||
|
||||
if len(artifactDataBlob) == 0 {
|
||||
bdw.Log.Error().Msgf("repodb: artifact data not found for digest %s", descriptor.Digest)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
var artifactData repodb.ArtifactData
|
||||
|
||||
err = json.Unmarshal(artifactDataBlob, &artifactData)
|
||||
if err != nil {
|
||||
bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal artifact data for digest %s", descriptor.Digest)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
manifestContent := ispec.Artifact{}
|
||||
|
||||
err := json.Unmarshal(artifactData.ManifestBlob, &manifestContent)
|
||||
if err != nil {
|
||||
bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal artifact manifest for digest %s", descriptor.Digest)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
referrerInfo = repodb.ReferrerInfo{
|
||||
Size: len(artifactData.ManifestBlob),
|
||||
Digest: descriptor.Digest,
|
||||
MediaType: manifestContent.MediaType,
|
||||
Annotations: manifestContent.Annotations,
|
||||
ArtifactType: manifestContent.ArtifactType,
|
||||
}
|
||||
}
|
||||
|
||||
if !common.MatchesArtifactTypes(referrerInfo.ArtifactType, artifactTypes) {
|
||||
continue
|
||||
}
|
||||
|
||||
referrersInfo = append(referrersInfo, referrerInfo)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return referrersInfo, err
|
||||
}
|
||||
|
||||
func (bdw *DBWrapper) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest,
|
||||
mediaType string,
|
||||
) error {
|
||||
|
@ -274,13 +564,16 @@ func (bdw *DBWrapper) SetRepoReference(repo string, reference string, manifestDi
|
|||
// object not found
|
||||
if len(repoMetaBlob) == 0 {
|
||||
var err error
|
||||
|
||||
repoMetaBlob, err = json.Marshal(repodb.RepoMetadata{
|
||||
// create a new object
|
||||
repoMeta := repodb.RepoMetadata{
|
||||
Name: repo,
|
||||
Tags: map[string]repodb.Descriptor{},
|
||||
Statistics: map[string]repodb.DescriptorStatistics{},
|
||||
Signatures: map[string]repodb.ManifestSignatures{},
|
||||
})
|
||||
Referrers: map[string][]repodb.Descriptor{},
|
||||
}
|
||||
|
||||
repoMetaBlob, err = json.Marshal(repoMeta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
@ -303,6 +596,7 @@ func (bdw *DBWrapper) SetRepoReference(repo string, reference string, manifestDi
|
|||
|
||||
repoMeta.Statistics[manifestDigest.String()] = repodb.DescriptorStatistics{DownloadCount: 0}
|
||||
repoMeta.Signatures[manifestDigest.String()] = repodb.ManifestSignatures{}
|
||||
repoMeta.Referrers[manifestDigest.String()] = []repodb.Descriptor{}
|
||||
|
||||
repoMetaBlob, err = json.Marshal(repoMeta)
|
||||
if err != nil {
|
||||
|
|
|
@ -70,6 +70,142 @@ func TestWrapperErrors(t *testing.T) {
|
|||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("GetArtifactData", func() {
|
||||
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
|
||||
artifactBuck := tx.Bucket([]byte(repodb.ArtifactDataBucket))
|
||||
|
||||
return artifactBuck.Put([]byte("artifactDigest"), []byte("wrong json"))
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, err = boltdbWrapper.GetArtifactData("artifactDigest")
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("SetReferrer", func() {
|
||||
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
|
||||
repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
|
||||
|
||||
return repoBuck.Put([]byte("repo"), []byte("wrong json"))
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = boltdbWrapper.SetReferrer("repo", "ref", repodb.Descriptor{})
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("DeleteReferrer", func() {
|
||||
Convey("RepoMeta not found", func() {
|
||||
err := boltdbWrapper.DeleteReferrer("r", "dig", "dig")
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("bad repo meta blob", func() {
|
||||
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
|
||||
repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
|
||||
|
||||
return repoBuck.Put([]byte("repo"), []byte("wrong json"))
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = boltdbWrapper.DeleteReferrer("repo", "dig", "dig")
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
})
|
||||
|
||||
Convey("GetReferrers", func() {
|
||||
Convey("RepoMeta not found", func() {
|
||||
_, err := boltdbWrapper.GetReferrers("repo", "dig")
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("bad repo meta blob", func() {
|
||||
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
|
||||
repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
|
||||
|
||||
return repoBuck.Put([]byte("repo"), []byte("wrong json"))
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, err = boltdbWrapper.GetReferrers("repo", "dig")
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
})
|
||||
|
||||
Convey("GetFilteredReferrersInfo", func() {
|
||||
Convey("getReferrers fails", func() {
|
||||
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
|
||||
repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
|
||||
|
||||
return repoBuck.Put([]byte("repo"), []byte("wrong json"))
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, err = boltdbWrapper.GetFilteredReferrersInfo("repo", "", nil)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("unmarshal erorrs", func() {
|
||||
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
|
||||
manifestBuck := tx.Bucket([]byte(repodb.ManifestDataBucket))
|
||||
artifactBuck := tx.Bucket([]byte(repodb.ArtifactDataBucket))
|
||||
|
||||
err = manifestBuck.Put([]byte("manifestDataRef"), []byte("bad json"))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = artifactBuck.Put([]byte("artifactDataRef"), []byte("bad json"))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
badBlob, err := json.Marshal(repodb.ArtifactData{
|
||||
ManifestBlob: []byte("bad json"),
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = artifactBuck.Put([]byte("artifactManifestRef"), badBlob)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
badBlob, err = json.Marshal(repodb.ManifestData{
|
||||
ManifestBlob: []byte("bad json"),
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = manifestBuck.Put([]byte("badManifest"), badBlob)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
return nil
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{
|
||||
Digest: "manifestDataRef",
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{
|
||||
Digest: "artifactDataRef",
|
||||
MediaType: ispec.MediaTypeArtifactManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{
|
||||
Digest: "badManifest",
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{
|
||||
Digest: "artifactManifestRef",
|
||||
MediaType: ispec.MediaTypeArtifactManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
refInfo, err := boltdbWrapper.GetFilteredReferrersInfo("repo", "refDigest", nil)
|
||||
So(err, ShouldBeNil)
|
||||
So(len(refInfo), ShouldEqual, 0)
|
||||
})
|
||||
})
|
||||
|
||||
Convey("SetRepoReference", func() {
|
||||
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
|
||||
repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
package common
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
|
@ -197,3 +198,38 @@ func containsString(strSlice []string, str string) bool {
|
|||
|
||||
return false
|
||||
}
|
||||
|
||||
func GetReferredSubject(descriptorBlob []byte) (godigest.Digest, bool) {
|
||||
var manifest ispec.Manifest
|
||||
|
||||
err := json.Unmarshal(descriptorBlob, &manifest)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
|
||||
if manifest.Subject == nil || manifest.Subject.Digest.String() == "" {
|
||||
return "", false
|
||||
}
|
||||
|
||||
return manifest.Subject.Digest, true
|
||||
}
|
||||
|
||||
func MatchesArtifactTypes(descriptorMediaType string, artifactTypes []string) bool {
|
||||
if len(artifactTypes) == 0 {
|
||||
return true
|
||||
}
|
||||
|
||||
found := false
|
||||
|
||||
for _, artifactType := range artifactTypes {
|
||||
if artifactType != "" && descriptorMediaType != artifactType {
|
||||
continue
|
||||
}
|
||||
|
||||
found = true
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
return found
|
||||
}
|
||||
|
|
24
pkg/meta/repodb/common/common_test.go
Normal file
24
pkg/meta/repodb/common/common_test.go
Normal file
|
@ -0,0 +1,24 @@
|
|||
package common_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
|
||||
"zotregistry.io/zot/pkg/meta/repodb/common"
|
||||
)
|
||||
|
||||
func TestUtils(t *testing.T) {
|
||||
Convey("GetReferredSubject", t, func() {
|
||||
_, err := common.GetReferredSubject([]byte("bad json"))
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("MatchesArtifactTypes", t, func() {
|
||||
res := common.MatchesArtifactTypes("", nil)
|
||||
So(res, ShouldBeTrue)
|
||||
|
||||
res = common.MatchesArtifactTypes("type", []string{"someOtherType"})
|
||||
So(res, ShouldBeFalse)
|
||||
})
|
||||
}
|
|
@ -29,6 +29,9 @@ func TestWrapperErrors(t *testing.T) {
|
|||
|
||||
repoMetaTablename := "RepoMetadataTable" + uuid.String()
|
||||
manifestDataTablename := "ManifestDataTable" + uuid.String()
|
||||
indexDataTablename := "IndexDataTable" + uuid.String()
|
||||
artifactDataTablename := "ArtifactDataTable" + uuid.String()
|
||||
|
||||
versionTablename := "Version" + uuid.String()
|
||||
|
||||
Convey("Create table errors", t, func() {
|
||||
|
@ -52,6 +55,8 @@ func TestWrapperErrors(t *testing.T) {
|
|||
Client: dynamodb.NewFromConfig(cfg),
|
||||
RepoMetaTablename: repoMetaTablename,
|
||||
ManifestDataTablename: manifestDataTablename,
|
||||
IndexDataTablename: indexDataTablename,
|
||||
ArtifactDataTablename: artifactDataTablename,
|
||||
VersionTablename: versionTablename,
|
||||
Patches: version.GetDynamoDBPatches(),
|
||||
Log: log.Logger{Logger: zerolog.New(os.Stdout)},
|
||||
|
@ -91,6 +96,8 @@ func TestWrapperErrors(t *testing.T) {
|
|||
RepoMetaTablename: repoMetaTablename,
|
||||
ManifestDataTablename: manifestDataTablename,
|
||||
VersionTablename: versionTablename,
|
||||
IndexDataTablename: indexDataTablename,
|
||||
ArtifactDataTablename: artifactDataTablename,
|
||||
Patches: version.GetDynamoDBPatches(),
|
||||
Log: log.Logger{Logger: zerolog.New(os.Stdout)},
|
||||
}
|
||||
|
|
|
@ -25,6 +25,8 @@ import (
|
|||
"zotregistry.io/zot/pkg/test"
|
||||
)
|
||||
|
||||
const badTablename = "bad tablename"
|
||||
|
||||
func TestIterator(t *testing.T) {
|
||||
const (
|
||||
endpoint = "http://localhost:4566"
|
||||
|
@ -40,6 +42,7 @@ func TestIterator(t *testing.T) {
|
|||
manifestDataTablename := "ManifestDataTable" + uuid.String()
|
||||
versionTablename := "Version" + uuid.String()
|
||||
indexDataTablename := "IndexDataTable" + uuid.String()
|
||||
artifactDataTablename := "ArtifactDataTable" + uuid.String()
|
||||
|
||||
Convey("TestIterator", t, func() {
|
||||
dynamoWrapper, err := dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{
|
||||
|
@ -48,6 +51,7 @@ func TestIterator(t *testing.T) {
|
|||
RepoMetaTablename: repoMetaTablename,
|
||||
ManifestDataTablename: manifestDataTablename,
|
||||
IndexDataTablename: indexDataTablename,
|
||||
ArtifactDataTablename: artifactDataTablename,
|
||||
VersionTablename: versionTablename,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
@ -133,6 +137,7 @@ func TestWrapperErrors(t *testing.T) {
|
|||
manifestDataTablename := "ManifestDataTable" + uuid.String()
|
||||
versionTablename := "Version" + uuid.String()
|
||||
indexDataTablename := "IndexDataTable" + uuid.String()
|
||||
artifactDataTablename := "ArtifactData" + uuid.String()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
|
@ -143,6 +148,7 @@ func TestWrapperErrors(t *testing.T) {
|
|||
RepoMetaTablename: repoMetaTablename,
|
||||
ManifestDataTablename: manifestDataTablename,
|
||||
IndexDataTablename: indexDataTablename,
|
||||
ArtifactDataTablename: artifactDataTablename,
|
||||
VersionTablename: versionTablename,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
@ -222,6 +228,114 @@ func TestWrapperErrors(t *testing.T) {
|
|||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("GetArtifactData", func() {
|
||||
dynamoWrapper.ArtifactDataTablename = badTablename
|
||||
_, err = dynamoWrapper.GetArtifactData("dig")
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("GetArtifactData unmarhsal error", func() {
|
||||
err = setBadArtifactData(dynamoWrapper.Client, artifactDataTablename, "dig")
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, err = dynamoWrapper.GetArtifactData("dig")
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("SetReferrer client error", func() {
|
||||
dynamoWrapper.RepoMetaTablename = badTablename
|
||||
err := dynamoWrapper.SetReferrer("repo", "", repodb.Descriptor{})
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("SetReferrer bad repoMeta", func() {
|
||||
err := setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo")
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = dynamoWrapper.SetReferrer("repo", "", repodb.Descriptor{})
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("GetReferrers client error", func() {
|
||||
dynamoWrapper.RepoMetaTablename = badTablename
|
||||
_, err := dynamoWrapper.GetReferrers("repo", "")
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("GetReferrers bad repoMeta", func() {
|
||||
err := setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo")
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, err = dynamoWrapper.GetReferrers("repo", "")
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("DeleteReferrer client error", func() {
|
||||
dynamoWrapper.RepoMetaTablename = badTablename
|
||||
err := dynamoWrapper.DeleteReferrer("repo", "", "")
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("DeleteReferrer bad repoMeta", func() {
|
||||
err := setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo")
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = dynamoWrapper.DeleteReferrer("repo", "", "")
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("GetFilteredReferrersInfo GetReferrers errors", func() {
|
||||
dynamoWrapper.RepoMetaTablename = badTablename
|
||||
_, err := dynamoWrapper.GetFilteredReferrersInfo("repo", "", nil)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("GetFilteredReferrersInfo getData fails", func() {
|
||||
dynamoWrapper.ManifestDataTablename = badTablename
|
||||
dynamoWrapper.ArtifactDataTablename = badTablename
|
||||
err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{
|
||||
Digest: "dig1",
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{
|
||||
Digest: "dig2",
|
||||
MediaType: ispec.MediaTypeArtifactManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, err := dynamoWrapper.GetFilteredReferrersInfo("repo", "rf", nil)
|
||||
So(err, ShouldBeNil)
|
||||
})
|
||||
|
||||
Convey("GetFilteredReferrersInfo bad descriptor blob", func() {
|
||||
err = dynamoWrapper.SetArtifactData("dig2", repodb.ArtifactData{
|
||||
ManifestBlob: []byte("bad json"),
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = dynamoWrapper.SetManifestData("dig3", repodb.ManifestData{
|
||||
ManifestBlob: []byte("bad json"),
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{
|
||||
Digest: "dig2",
|
||||
MediaType: ispec.MediaTypeArtifactManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{
|
||||
Digest: "dig3",
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, err := dynamoWrapper.GetFilteredReferrersInfo("repo", "rf", nil)
|
||||
So(err, ShouldBeNil)
|
||||
})
|
||||
|
||||
Convey("IncrementRepoStars GetRepoMeta error", func() {
|
||||
err = dynamoWrapper.IncrementRepoStars("repo")
|
||||
So(err, ShouldNotBeNil)
|
||||
|
@ -688,6 +802,7 @@ func TestWrapperErrors(t *testing.T) {
|
|||
RepoMetaTablename: "",
|
||||
ManifestDataTablename: manifestDataTablename,
|
||||
IndexDataTablename: indexDataTablename,
|
||||
ArtifactDataTablename: artifactDataTablename,
|
||||
VersionTablename: versionTablename,
|
||||
})
|
||||
So(err, ShouldNotBeNil)
|
||||
|
@ -698,6 +813,7 @@ func TestWrapperErrors(t *testing.T) {
|
|||
RepoMetaTablename: repoMetaTablename,
|
||||
ManifestDataTablename: "",
|
||||
IndexDataTablename: indexDataTablename,
|
||||
ArtifactDataTablename: artifactDataTablename,
|
||||
VersionTablename: versionTablename,
|
||||
})
|
||||
So(err, ShouldNotBeNil)
|
||||
|
@ -708,6 +824,7 @@ func TestWrapperErrors(t *testing.T) {
|
|||
RepoMetaTablename: repoMetaTablename,
|
||||
ManifestDataTablename: manifestDataTablename,
|
||||
IndexDataTablename: "",
|
||||
ArtifactDataTablename: artifactDataTablename,
|
||||
VersionTablename: versionTablename,
|
||||
})
|
||||
So(err, ShouldNotBeNil)
|
||||
|
@ -718,6 +835,7 @@ func TestWrapperErrors(t *testing.T) {
|
|||
RepoMetaTablename: repoMetaTablename,
|
||||
ManifestDataTablename: manifestDataTablename,
|
||||
IndexDataTablename: indexDataTablename,
|
||||
ArtifactDataTablename: artifactDataTablename,
|
||||
VersionTablename: "",
|
||||
})
|
||||
So(err, ShouldNotBeNil)
|
||||
|
@ -728,8 +846,20 @@ func TestWrapperErrors(t *testing.T) {
|
|||
RepoMetaTablename: repoMetaTablename,
|
||||
ManifestDataTablename: manifestDataTablename,
|
||||
IndexDataTablename: indexDataTablename,
|
||||
ArtifactDataTablename: "",
|
||||
VersionTablename: versionTablename,
|
||||
})
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
_, err = dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck
|
||||
Endpoint: endpoint,
|
||||
Region: region,
|
||||
RepoMetaTablename: repoMetaTablename,
|
||||
ManifestDataTablename: manifestDataTablename,
|
||||
IndexDataTablename: indexDataTablename,
|
||||
VersionTablename: versionTablename,
|
||||
ArtifactDataTablename: artifactDataTablename,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
})
|
||||
}
|
||||
|
@ -759,6 +889,31 @@ func setBadManifestData(client *dynamodb.Client, manifestDataTableName, digest s
|
|||
return err
|
||||
}
|
||||
|
||||
func setBadArtifactData(client *dynamodb.Client, artifactDataTablename, digest string) error {
|
||||
mdAttributeValue, err := attributevalue.Marshal("string")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{
|
||||
ExpressionAttributeNames: map[string]string{
|
||||
"#AD": "ArtifactData",
|
||||
},
|
||||
ExpressionAttributeValues: map[string]types.AttributeValue{
|
||||
":ArtifactData": mdAttributeValue,
|
||||
},
|
||||
Key: map[string]types.AttributeValue{
|
||||
"ArtifactDigest": &types.AttributeValueMemberS{
|
||||
Value: digest,
|
||||
},
|
||||
},
|
||||
TableName: aws.String(artifactDataTablename),
|
||||
UpdateExpression: aws.String("SET #AD = :ArtifactData"),
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func setBadIndexData(client *dynamodb.Client, indexDataTableName, digest string) error {
|
||||
mdAttributeValue, err := attributevalue.Marshal("string")
|
||||
if err != nil {
|
||||
|
|
|
@ -32,6 +32,7 @@ type DBWrapper struct {
|
|||
RepoMetaTablename string
|
||||
IndexDataTablename string
|
||||
ManifestDataTablename string
|
||||
ArtifactDataTablename string
|
||||
VersionTablename string
|
||||
Patches []func(client *dynamodb.Client, tableNames map[string]string) error
|
||||
Log log.Logger
|
||||
|
@ -62,6 +63,7 @@ func NewDynamoDBWrapper(params dynamoParams.DBDriverParameters) (*DBWrapper, err
|
|||
RepoMetaTablename: params.RepoMetaTablename,
|
||||
ManifestDataTablename: params.ManifestDataTablename,
|
||||
IndexDataTablename: params.IndexDataTablename,
|
||||
ArtifactDataTablename: params.ArtifactDataTablename,
|
||||
VersionTablename: params.VersionTablename,
|
||||
Patches: version.GetDynamoDBPatches(),
|
||||
Log: log.Logger{Logger: zerolog.New(os.Stdout)},
|
||||
|
@ -82,6 +84,11 @@ func NewDynamoDBWrapper(params dynamoParams.DBDriverParameters) (*DBWrapper, err
|
|||
return nil, err
|
||||
}
|
||||
|
||||
err = dynamoWrapper.createArtifactDataTable()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = dynamoWrapper.createIndexDataTable()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
@ -158,6 +165,7 @@ func (dwr *DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Diges
|
|||
Tags: map[string]repodb.Descriptor{},
|
||||
Statistics: map[string]repodb.DescriptorStatistics{},
|
||||
Signatures: map[string]repodb.ManifestSignatures{},
|
||||
Referrers: map[string][]repodb.Descriptor{},
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -307,6 +315,246 @@ func (dwr *DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexDat
|
|||
return indexData, nil
|
||||
}
|
||||
|
||||
func (dwr DBWrapper) SetArtifactData(artifactDigest godigest.Digest, artifactData repodb.ArtifactData) error {
|
||||
artifactAttributeValue, err := attributevalue.Marshal(artifactData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{
|
||||
ExpressionAttributeNames: map[string]string{
|
||||
"#AD": "ArtifactData",
|
||||
},
|
||||
ExpressionAttributeValues: map[string]types.AttributeValue{
|
||||
":ArtifactData": artifactAttributeValue,
|
||||
},
|
||||
Key: map[string]types.AttributeValue{
|
||||
"ArtifactDigest": &types.AttributeValueMemberS{
|
||||
Value: artifactDigest.String(),
|
||||
},
|
||||
},
|
||||
TableName: aws.String(dwr.ArtifactDataTablename),
|
||||
UpdateExpression: aws.String("SET #AD = :ArtifactData"),
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (dwr DBWrapper) GetArtifactData(artifactDigest godigest.Digest) (repodb.ArtifactData, error) {
|
||||
resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{
|
||||
TableName: aws.String(dwr.ArtifactDataTablename),
|
||||
Key: map[string]types.AttributeValue{
|
||||
"ArtifactDigest": &types.AttributeValueMemberS{
|
||||
Value: artifactDigest.String(),
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return repodb.ArtifactData{}, err
|
||||
}
|
||||
|
||||
if resp.Item == nil {
|
||||
return repodb.ArtifactData{}, zerr.ErrRepoMetaNotFound
|
||||
}
|
||||
|
||||
var artifactData repodb.ArtifactData
|
||||
|
||||
err = attributevalue.Unmarshal(resp.Item["ArtifactData"], &artifactData)
|
||||
if err != nil {
|
||||
return repodb.ArtifactData{}, err
|
||||
}
|
||||
|
||||
return artifactData, nil
|
||||
}
|
||||
|
||||
func (dwr DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error {
|
||||
resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{
|
||||
TableName: aws.String(dwr.RepoMetaTablename),
|
||||
Key: map[string]types.AttributeValue{
|
||||
"RepoName": &types.AttributeValueMemberS{Value: repo},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
repoMeta := repodb.RepoMetadata{
|
||||
Name: repo,
|
||||
Tags: map[string]repodb.Descriptor{},
|
||||
Statistics: map[string]repodb.DescriptorStatistics{},
|
||||
Signatures: map[string]repodb.ManifestSignatures{},
|
||||
Referrers: map[string][]repodb.Descriptor{},
|
||||
}
|
||||
|
||||
if resp.Item != nil {
|
||||
err := attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
refferers := repoMeta.Referrers[referredDigest.String()]
|
||||
|
||||
for i := range refferers {
|
||||
if refferers[i].Digest == referrer.Digest {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
refferers = append(refferers, referrer)
|
||||
|
||||
repoMeta.Referrers[referredDigest.String()] = refferers
|
||||
|
||||
return dwr.setRepoMeta(repo, repoMeta)
|
||||
}
|
||||
|
||||
func (dwr DBWrapper) GetReferrers(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error) {
|
||||
resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{
|
||||
TableName: aws.String(dwr.RepoMetaTablename),
|
||||
Key: map[string]types.AttributeValue{
|
||||
"RepoName": &types.AttributeValueMemberS{Value: repo},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return []repodb.Descriptor{}, err
|
||||
}
|
||||
|
||||
repoMeta := repodb.RepoMetadata{
|
||||
Name: repo,
|
||||
Tags: map[string]repodb.Descriptor{},
|
||||
Statistics: map[string]repodb.DescriptorStatistics{},
|
||||
Signatures: map[string]repodb.ManifestSignatures{},
|
||||
Referrers: map[string][]repodb.Descriptor{},
|
||||
}
|
||||
|
||||
if resp.Item != nil {
|
||||
err := attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta)
|
||||
if err != nil {
|
||||
return []repodb.Descriptor{}, err
|
||||
}
|
||||
}
|
||||
|
||||
return repoMeta.Referrers[referredDigest.String()], nil
|
||||
}
|
||||
|
||||
func (dwr DBWrapper) DeleteReferrer(repo string, referredDigest godigest.Digest,
|
||||
referrerDigest godigest.Digest,
|
||||
) error {
|
||||
resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{
|
||||
TableName: aws.String(dwr.RepoMetaTablename),
|
||||
Key: map[string]types.AttributeValue{
|
||||
"RepoName": &types.AttributeValueMemberS{Value: repo},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
repoMeta := repodb.RepoMetadata{
|
||||
Name: repo,
|
||||
Tags: map[string]repodb.Descriptor{},
|
||||
Statistics: map[string]repodb.DescriptorStatistics{},
|
||||
Signatures: map[string]repodb.ManifestSignatures{},
|
||||
Referrers: map[string][]repodb.Descriptor{},
|
||||
}
|
||||
|
||||
if resp.Item != nil {
|
||||
err := attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
referrers := repoMeta.Referrers[referredDigest.String()]
|
||||
|
||||
for i := range referrers {
|
||||
if referrers[i].Digest == referrerDigest.String() {
|
||||
referrers = append(referrers[:i], referrers[i+1:]...)
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
repoMeta.Referrers[referredDigest.String()] = referrers
|
||||
|
||||
return dwr.setRepoMeta(repo, repoMeta)
|
||||
}
|
||||
|
||||
func (dwr DBWrapper) GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest,
|
||||
artifactTypes []string,
|
||||
) ([]repodb.ReferrerInfo, error) {
|
||||
referrersDescriptors, err := dwr.GetReferrers(repo, referredDigest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
referrersInfo := []repodb.ReferrerInfo{}
|
||||
|
||||
for _, descriptor := range referrersDescriptors {
|
||||
referrerInfo := repodb.ReferrerInfo{}
|
||||
|
||||
switch descriptor.MediaType {
|
||||
case ispec.MediaTypeImageManifest:
|
||||
manifestData, err := dwr.GetManifestData(godigest.Digest(descriptor.Digest))
|
||||
if err != nil {
|
||||
dwr.Log.Error().Msgf("repodb: manifest data not found for digest %s", descriptor.Digest)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
var manifestContent ispec.Manifest
|
||||
|
||||
err = json.Unmarshal(manifestData.ManifestBlob, &manifestContent)
|
||||
if err != nil {
|
||||
dwr.Log.Error().Err(err).Msgf("repodb: can't unmarhsal manifest for digest %s",
|
||||
descriptor.Digest)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
referrerInfo = repodb.ReferrerInfo{
|
||||
Digest: descriptor.Digest,
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
ArtifactType: manifestContent.Config.MediaType,
|
||||
Size: len(manifestData.ManifestBlob),
|
||||
Annotations: manifestContent.Annotations,
|
||||
}
|
||||
case ispec.MediaTypeArtifactManifest:
|
||||
artifactData, err := dwr.GetArtifactData(godigest.Digest(descriptor.Digest))
|
||||
if err != nil {
|
||||
dwr.Log.Error().Msgf("repodb: artifact data not found for digest %s", descriptor.Digest)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
manifestContent := ispec.Artifact{}
|
||||
|
||||
err = json.Unmarshal(artifactData.ManifestBlob, &manifestContent)
|
||||
if err != nil {
|
||||
dwr.Log.Error().Err(err).Msgf("repodb: can't unmarhsal artifact manifest for digest %s", descriptor.Digest)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
referrerInfo = repodb.ReferrerInfo{
|
||||
Digest: descriptor.Digest,
|
||||
MediaType: manifestContent.MediaType,
|
||||
ArtifactType: manifestContent.ArtifactType,
|
||||
Size: len(artifactData.ManifestBlob),
|
||||
Annotations: manifestContent.Annotations,
|
||||
}
|
||||
}
|
||||
|
||||
if !common.MatchesArtifactTypes(referrerInfo.ArtifactType, artifactTypes) {
|
||||
continue
|
||||
}
|
||||
|
||||
referrersInfo = append(referrersInfo, referrerInfo)
|
||||
}
|
||||
|
||||
return referrersInfo, nil
|
||||
}
|
||||
|
||||
func (dwr *DBWrapper) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest,
|
||||
mediaType string,
|
||||
) error {
|
||||
|
@ -329,6 +577,7 @@ func (dwr *DBWrapper) SetRepoReference(repo string, reference string, manifestDi
|
|||
Tags: map[string]repodb.Descriptor{},
|
||||
Statistics: map[string]repodb.DescriptorStatistics{},
|
||||
Signatures: map[string]repodb.ManifestSignatures{},
|
||||
Referrers: map[string][]repodb.Descriptor{},
|
||||
}
|
||||
|
||||
if resp.Item != nil {
|
||||
|
@ -347,6 +596,7 @@ func (dwr *DBWrapper) SetRepoReference(repo string, reference string, manifestDi
|
|||
|
||||
repoMeta.Statistics[manifestDigest.String()] = repodb.DescriptorStatistics{DownloadCount: 0}
|
||||
repoMeta.Signatures[manifestDigest.String()] = repodb.ManifestSignatures{}
|
||||
repoMeta.Referrers[manifestDigest.String()] = []repodb.Descriptor{}
|
||||
|
||||
err = dwr.setRepoMeta(repo, repoMeta)
|
||||
|
||||
|
@ -1392,6 +1642,31 @@ func (dwr *DBWrapper) createIndexDataTable() error {
|
|||
return dwr.waitTableToBeCreated(dwr.IndexDataTablename)
|
||||
}
|
||||
|
||||
func (dwr DBWrapper) createArtifactDataTable() error {
|
||||
_, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{
|
||||
TableName: aws.String(dwr.ArtifactDataTablename),
|
||||
AttributeDefinitions: []types.AttributeDefinition{
|
||||
{
|
||||
AttributeName: aws.String("ArtifactDigest"),
|
||||
AttributeType: types.ScalarAttributeTypeS,
|
||||
},
|
||||
},
|
||||
KeySchema: []types.KeySchemaElement{
|
||||
{
|
||||
AttributeName: aws.String("ArtifactDigest"),
|
||||
KeyType: types.KeyTypeHash,
|
||||
},
|
||||
},
|
||||
BillingMode: types.BillingModePayPerRequest,
|
||||
})
|
||||
|
||||
if err != nil && !strings.Contains(err.Error(), "Table already exists") {
|
||||
return err
|
||||
}
|
||||
|
||||
return dwr.waitTableToBeCreated(dwr.ManifestDataTablename)
|
||||
}
|
||||
|
||||
func (dwr *DBWrapper) createVersionTable() error {
|
||||
_, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{
|
||||
TableName: aws.String(dwr.VersionTablename),
|
||||
|
|
|
@ -2,5 +2,5 @@ package params
|
|||
|
||||
type DBDriverParameters struct {
|
||||
Endpoint, Region, RepoMetaTablename, ManifestDataTablename, IndexDataTablename,
|
||||
VersionTablename string
|
||||
ArtifactDataTablename, VersionTablename string
|
||||
}
|
||||
|
|
|
@ -12,20 +12,21 @@ import (
|
|||
"zotregistry.io/zot/pkg/storage"
|
||||
)
|
||||
|
||||
// SyncRepoDB will sync all repos found in the rootdirectory of the oci layout that zot was deployed on.
|
||||
func SyncRepoDB(repoDB RepoDB, storeController storage.StoreController, log log.Logger) error {
|
||||
// ParseStorage will sync all repos found in the rootdirectory of the oci layout that zot was deployed on with the
|
||||
// ParseStorage database.
|
||||
func ParseStorage(repoDB RepoDB, storeController storage.StoreController, log log.Logger) error {
|
||||
allRepos, err := getAllRepos(storeController)
|
||||
if err != nil {
|
||||
rootDir := storeController.DefaultStore.RootDir()
|
||||
log.Error().Err(err).Msgf("sync-repodb: failed to get all repo names present under %s", rootDir)
|
||||
log.Error().Err(err).Msgf("load-local-layout: failed to get all repo names present under %s", rootDir)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
for _, repo := range allRepos {
|
||||
err := SyncRepo(repo, repoDB, storeController, log)
|
||||
err := ParseRepo(repo, repoDB, storeController, log)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("sync-repodb: failed to sync repo %s", repo)
|
||||
log.Error().Err(err).Msgf("load-local-layout: failed to sync repo %s", repo)
|
||||
|
||||
return err
|
||||
}
|
||||
|
@ -34,13 +35,13 @@ func SyncRepoDB(repoDB RepoDB, storeController storage.StoreController, log log.
|
|||
return nil
|
||||
}
|
||||
|
||||
// SyncRepo reads the contents of a repo and syncs all images signatures found.
|
||||
func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreController, log log.Logger) error {
|
||||
// ParseRepo reads the contents of a repo and syncs all images and signatures found.
|
||||
func ParseRepo(repo string, repoDB RepoDB, storeController storage.StoreController, log log.Logger) error {
|
||||
imageStore := storeController.GetImageStore(repo)
|
||||
|
||||
indexBlob, err := imageStore.GetIndexContent(repo)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("sync-repo: failed to read index.json for repo %s", repo)
|
||||
log.Error().Err(err).Msgf("load-repo: failed to read index.json for repo %s", repo)
|
||||
|
||||
return err
|
||||
}
|
||||
|
@ -49,14 +50,14 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
|
|||
|
||||
err = json.Unmarshal(indexBlob, &indexContent)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("sync-repo: failed to unmarshal index.json for repo %s", repo)
|
||||
log.Error().Err(err).Msgf("load-repo: failed to unmarshal index.json for repo %s", repo)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
err = resetRepoMetaTags(repo, repoDB, log)
|
||||
if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) {
|
||||
log.Error().Err(err).Msgf("sync-repo: failed to reset tag field in RepoMetadata for repo %s", repo)
|
||||
log.Error().Err(err).Msgf("load-repo: failed to reset tag field in RepoMetadata for repo %s", repo)
|
||||
|
||||
return err
|
||||
}
|
||||
|
@ -76,7 +77,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
|
|||
|
||||
manifestMetaIsPresent, err := isManifestMetaPresent(repo, manifest, repoDB)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("sync-repo: error checking manifestMeta in RepoDB")
|
||||
log.Error().Err(err).Msgf("load-repo: error checking manifestMeta in RepoDB")
|
||||
|
||||
return err
|
||||
}
|
||||
|
@ -84,7 +85,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
|
|||
if manifestMetaIsPresent && hasTag {
|
||||
err = repoDB.SetRepoReference(repo, tag, manifest.Digest, manifest.MediaType)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("sync-repo: failed to set repo tag for %s:%s", repo, tag)
|
||||
log.Error().Err(err).Msgf("load-repo: failed to set repo tag for %s:%s", repo, tag)
|
||||
|
||||
return err
|
||||
}
|
||||
|
@ -94,7 +95,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
|
|||
|
||||
manifestBlob, digest, _, err := imageStore.GetImageManifest(repo, manifest.Digest.String())
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("sync-repo: failed to set repo tag for %s:%s", repo, tag)
|
||||
log.Error().Err(err).Msgf("load-repo: failed to set repo tag for %s:%s", repo, tag)
|
||||
|
||||
return err
|
||||
}
|
||||
|
@ -105,7 +106,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
|
|||
if errors.Is(err, zerr.ErrOrphanSignature) {
|
||||
continue
|
||||
} else {
|
||||
log.Error().Err(err).Msgf("sync-repo: failed checking if image is signature for %s:%s", repo, tag)
|
||||
log.Error().Err(err).Msgf("load-repo: failed checking if image is signature for %s:%s", repo, tag)
|
||||
|
||||
return err
|
||||
}
|
||||
|
@ -134,7 +135,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
|
|||
err = SetMetadataFromInput(repo, reference, manifest.MediaType, manifest.Digest, manifestBlob,
|
||||
imageStore, repoDB, log)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("sync-repo: failed to set metadata for %s:%s", repo, tag)
|
||||
log.Error().Err(err).Msgf("load-repo: failed to set metadata for %s:%s", repo, tag)
|
||||
|
||||
return err
|
||||
}
|
||||
|
@ -142,12 +143,13 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
|
|||
|
||||
// manage the signatures found
|
||||
for _, sigData := range signaturesFound {
|
||||
err := repoDB.AddManifestSignature(repo, godigest.Digest(sigData.signedManifestDigest), SignatureMetadata{
|
||||
SignatureType: sigData.signatureType,
|
||||
SignatureDigest: sigData.signatureDigest,
|
||||
})
|
||||
err := repoDB.AddManifestSignature(repo, godigest.Digest(sigData.signedManifestDigest),
|
||||
SignatureMetadata{
|
||||
SignatureType: sigData.signatureType,
|
||||
SignatureDigest: sigData.signatureDigest,
|
||||
})
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("sync-repo: failed set signature meta for signed image %s:%s manifest digest %s ",
|
||||
log.Error().Err(err).Msgf("load-repo: failed set signature meta for signed image %s:%s manifest digest %s ",
|
||||
sigData.repo, sigData.tag, sigData.signedManifestDigest)
|
||||
|
||||
return err
|
||||
|
@ -161,13 +163,13 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
|
|||
func resetRepoMetaTags(repo string, repoDB RepoDB, log log.Logger) error {
|
||||
repoMeta, err := repoDB.GetRepoMeta(repo)
|
||||
if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) {
|
||||
log.Error().Err(err).Msgf("sync-repo: failed to get RepoMeta for repo %s", repo)
|
||||
log.Error().Err(err).Msgf("load-repo: failed to get RepoMeta for repo %s", repo)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
if errors.Is(err, zerr.ErrRepoMetaNotFound) {
|
||||
log.Info().Msgf("sync-repo: RepoMeta not found for repo %s, new RepoMeta will be created", repo)
|
||||
log.Info().Msgf("load-repo: RepoMeta not found for repo %s, new RepoMeta will be created", repo)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
@ -176,7 +178,7 @@ func resetRepoMetaTags(repo string, repoDB RepoDB, log log.Logger) error {
|
|||
// We should have a way to delete all tags at once
|
||||
err := repoDB.DeleteRepoTag(repo, tag)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msgf("sync-repo: failed to delete tag %s from RepoMeta for repo %s", tag, repo)
|
||||
log.Error().Err(err).Msgf("load-repo: failed to delete tag %s from RepoMeta for repo %s", tag, repo)
|
||||
|
||||
return err
|
||||
}
|
||||
|
@ -220,7 +222,7 @@ func isManifestMetaPresent(repo string, manifest ispec.Descriptor, repoDB RepoDB
|
|||
}
|
||||
|
||||
// NewManifestMeta takes raw data about an image and createa a new ManifestMetadate object.
|
||||
func NewManifestData(repoName string, manifestBlob []byte, imgStore storage.ImageStore,
|
||||
func NewManifestData(repoName string, manifestBlob []byte, imageStore storage.ImageStore,
|
||||
) (ManifestData, error) {
|
||||
var (
|
||||
manifestContent ispec.Manifest
|
||||
|
@ -233,7 +235,7 @@ func NewManifestData(repoName string, manifestBlob []byte, imgStore storage.Imag
|
|||
return ManifestData{}, err
|
||||
}
|
||||
|
||||
configBlob, err := imgStore.GetBlobContent(repoName, manifestContent.Config.Digest)
|
||||
configBlob, err := imageStore.GetBlobContent(repoName, manifestContent.Config.Digest)
|
||||
if err != nil {
|
||||
return ManifestData{}, err
|
||||
}
|
||||
|
@ -249,7 +251,7 @@ func NewManifestData(repoName string, manifestBlob []byte, imgStore storage.Imag
|
|||
return manifestData, nil
|
||||
}
|
||||
|
||||
func NewIndexData(repoName string, indexBlob []byte,
|
||||
func NewIndexData(repoName string, indexBlob []byte, imageStore storage.ImageStore,
|
||||
) IndexData {
|
||||
indexData := IndexData{}
|
||||
|
||||
|
@ -258,6 +260,13 @@ func NewIndexData(repoName string, indexBlob []byte,
|
|||
return indexData
|
||||
}
|
||||
|
||||
func NewArtifactData(repo string, descriptorBlob []byte, imageStore storage.ImageStore,
|
||||
) ArtifactData {
|
||||
return ArtifactData{
|
||||
ManifestBlob: descriptorBlob,
|
||||
}
|
||||
}
|
||||
|
||||
// SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag
|
||||
// (in case the reference is a tag). The function expects image manifests and indexes (multi arch images).
|
||||
func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Digest, descriptorBlob []byte,
|
||||
|
@ -277,12 +286,33 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig
|
|||
return err
|
||||
}
|
||||
case ispec.MediaTypeImageIndex:
|
||||
indexData := NewIndexData(repo, descriptorBlob)
|
||||
indexData := NewIndexData(repo, descriptorBlob, imageStore)
|
||||
|
||||
err := repoDB.SetIndexData(digest, indexData)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("repodb: error while putting index data")
|
||||
|
||||
return err
|
||||
}
|
||||
case ispec.MediaTypeArtifactManifest:
|
||||
artifactData := NewArtifactData(repo, descriptorBlob, imageStore)
|
||||
|
||||
err := repoDB.SetArtifactData(digest, artifactData)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("repodb: error while putting artifact data")
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if refferredDigest, hasSubject := GetReferredSubject(descriptorBlob); hasSubject {
|
||||
err := repoDB.SetReferrer(repo, refferredDigest, Descriptor{
|
||||
Digest: digest.String(),
|
||||
MediaType: mediaType,
|
||||
})
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("repodb: error while settingg referrer")
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
@ -296,3 +326,18 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig
|
|||
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetReferredSubject(descriptorBlob []byte) (godigest.Digest, bool) {
|
||||
var manifest ispec.Manifest
|
||||
|
||||
err := json.Unmarshal(descriptorBlob, &manifest)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
|
||||
if manifest.Subject == nil || manifest.Subject.Digest.String() == "" {
|
||||
return "", false
|
||||
}
|
||||
|
||||
return manifest.Subject.Digest, true
|
||||
}
|
|
@ -12,7 +12,6 @@ import (
|
|||
|
||||
godigest "github.com/opencontainers/go-digest"
|
||||
ispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
oras "github.com/oras-project/artifacts-spec/specs-go/v1"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
|
||||
zerr "zotregistry.io/zot/errors"
|
||||
|
@ -32,8 +31,8 @@ const repo = "repo"
|
|||
|
||||
var ErrTestError = errors.New("test error")
|
||||
|
||||
func TestSyncRepoDBErrors(t *testing.T) {
|
||||
Convey("SyncRepoDB", t, func() {
|
||||
func TestLoadOCILayoutErrors(t *testing.T) {
|
||||
Convey("LoadOCILayout", t, func() {
|
||||
imageStore := mocks.MockedImageStore{
|
||||
GetIndexContentFn: func(repo string) ([]byte, error) {
|
||||
return nil, ErrTestError
|
||||
|
@ -46,7 +45,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
|
|||
repoDB := mocks.RepoDBMock{}
|
||||
|
||||
// sync repo fail
|
||||
err := repodb.SyncRepoDB(repoDB, storeController, log.NewLogger("debug", ""))
|
||||
err := repodb.ParseStorage(repoDB, storeController, log.NewLogger("debug", ""))
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
Convey("getAllRepos errors", func() {
|
||||
|
@ -67,12 +66,12 @@ func TestSyncRepoDBErrors(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
err := repodb.SyncRepoDB(repoDB, storeController, log.NewLogger("debug", ""))
|
||||
err := repodb.ParseStorage(repoDB, storeController, log.NewLogger("debug", ""))
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
})
|
||||
|
||||
Convey("SyncRepo", t, func() {
|
||||
Convey("LoadRepo", t, func() {
|
||||
imageStore := mocks.MockedImageStore{}
|
||||
storeController := storage.StoreController{DefaultStore: &imageStore}
|
||||
repoDB := mocks.RepoDBMock{}
|
||||
|
@ -83,7 +82,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
|
|||
return nil, ErrTestError
|
||||
}
|
||||
|
||||
err := repodb.SyncRepo("repo", repoDB, storeController, log)
|
||||
err := repodb.ParseRepo("repo", repoDB, storeController, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
|
@ -92,7 +91,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
|
|||
return []byte("Invalid JSON"), nil
|
||||
}
|
||||
|
||||
err := repodb.SyncRepo("repo", repoDB, storeController, log)
|
||||
err := repodb.ParseRepo("repo", repoDB, storeController, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
|
@ -106,7 +105,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
|
|||
return repodb.RepoMetadata{}, ErrTestError
|
||||
}
|
||||
|
||||
err := repodb.SyncRepo("repo", repoDB, storeController, log)
|
||||
err := repodb.ParseRepo("repo", repoDB, storeController, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
|
@ -125,7 +124,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
|
|||
return ErrTestError
|
||||
}
|
||||
|
||||
err := repodb.SyncRepo("repo", repoDB, storeController, log)
|
||||
err := repodb.ParseRepo("repo", repoDB, storeController, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
})
|
||||
|
@ -154,7 +153,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
|
|||
return repodb.ManifestMetadata{}, ErrTestError
|
||||
}
|
||||
|
||||
err = repodb.SyncRepo("repo", repoDB, storeController, log)
|
||||
err = repodb.ParseRepo("repo", repoDB, storeController, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
})
|
||||
|
@ -183,7 +182,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
|
|||
return ErrTestError
|
||||
}
|
||||
|
||||
err = repodb.SyncRepo("repo", repoDB, storeController, log)
|
||||
err = repodb.ParseRepo("repo", repoDB, storeController, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
})
|
||||
|
@ -215,7 +214,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
|
|||
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
|
||||
return nil, "", "", ErrTestError
|
||||
}
|
||||
err = repodb.SyncRepo("repo", repoDB, storeController, log)
|
||||
err = repodb.ParseRepo("repo", repoDB, storeController, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
|
@ -224,7 +223,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
|
|||
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
|
||||
return []byte("Invalid JSON"), "", "", nil
|
||||
}
|
||||
err = repodb.SyncRepo("repo", repoDB, storeController, log)
|
||||
err = repodb.ParseRepo("repo", repoDB, storeController, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
Convey("CheckIsImageSignature -> not signature", func() {
|
||||
|
@ -241,7 +240,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
|
|||
return nil, ErrTestError
|
||||
}
|
||||
|
||||
err = repodb.SyncRepo("repo", repoDB, storeController, log)
|
||||
err = repodb.ParseRepo("repo", repoDB, storeController, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
|
@ -250,17 +249,19 @@ func TestSyncRepoDBErrors(t *testing.T) {
|
|||
return []byte("invalid JSON"), nil
|
||||
}
|
||||
|
||||
err = repodb.SyncRepo("repo", repoDB, storeController, log)
|
||||
err = repodb.ParseRepo("repo", repoDB, storeController, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
})
|
||||
|
||||
Convey("CheckIsImageSignature -> is signature", func() {
|
||||
manifestContent := oras.Manifest{
|
||||
Subject: &oras.Descriptor{
|
||||
manifestContent := ispec.Artifact{
|
||||
Subject: &ispec.Descriptor{
|
||||
Digest: "123",
|
||||
},
|
||||
ArtifactType: "application/vnd.cncf.notary.signature",
|
||||
}
|
||||
|
||||
manifestBlob, err := json.Marshal(manifestContent)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
|
@ -274,14 +275,14 @@ func TestSyncRepoDBErrors(t *testing.T) {
|
|||
return ErrTestError
|
||||
}
|
||||
|
||||
err = repodb.SyncRepo("repo", repoDB, storeController, log)
|
||||
err = repodb.ParseRepo("repo", repoDB, storeController, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestSyncRepoDBWithStorage(t *testing.T) {
|
||||
func TestLoadOCILayoutWithStorage(t *testing.T) {
|
||||
Convey("Boltdb", t, func() {
|
||||
rootDir := t.TempDir()
|
||||
|
||||
|
@ -359,7 +360,7 @@ func TestSyncRepoDBWithStorage(t *testing.T) {
|
|||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = repodb.SyncRepoDB(repoDB, storeController, log.NewLogger("debug", ""))
|
||||
err = repodb.ParseStorage(repoDB, storeController, log.NewLogger("debug", ""))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
repos, err := repoDB.GetMultipleRepoMeta(
|
||||
|
@ -435,7 +436,7 @@ func TestSyncRepoDBWithStorage(t *testing.T) {
|
|||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = repodb.SyncRepoDB(repoDB, storeController, log.NewLogger("debug", ""))
|
||||
err = repodb.ParseStorage(repoDB, storeController, log.NewLogger("debug", ""))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
repos, err := repoDB.GetMultipleRepoMeta(
|
||||
|
@ -451,7 +452,7 @@ func TestSyncRepoDBWithStorage(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func TestSyncRepoDBDynamoWrapper(t *testing.T) {
|
||||
func TestLoadOCILayoutDynamoWrapper(t *testing.T) {
|
||||
skipIt(t)
|
||||
|
||||
Convey("Dynamodb", t, func() {
|
||||
|
@ -532,6 +533,7 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
|
|||
RepoMetaTablename: "RepoMetadataTable",
|
||||
ManifestDataTablename: "ManifestDataTable",
|
||||
IndexDataTablename: "IndexDataTable",
|
||||
ArtifactDataTablename: "ArtifactDataTable",
|
||||
VersionTablename: "Version",
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
@ -542,7 +544,7 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
|
|||
err = dynamoWrapper.ResetRepoMetaTable()
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = repodb.SyncRepoDB(dynamoWrapper, storeController, log.NewLogger("debug", ""))
|
||||
err = repodb.ParseStorage(dynamoWrapper, storeController, log.NewLogger("debug", ""))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
repos, err := dynamoWrapper.GetMultipleRepoMeta(
|
||||
|
@ -618,12 +620,13 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
|
|||
Region: "us-east-2",
|
||||
RepoMetaTablename: "RepoMetadataTable",
|
||||
ManifestDataTablename: "ManifestDataTable",
|
||||
ArtifactDataTablename: "ArtifactDataTable",
|
||||
IndexDataTablename: "IndexDataTable",
|
||||
VersionTablename: "Version",
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = repodb.SyncRepoDB(repoDB, storeController, log.NewLogger("debug", ""))
|
||||
err = repodb.ParseStorage(repoDB, storeController, log.NewLogger("debug", ""))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
repos, err := repoDB.GetMultipleRepoMeta(
|
||||
|
@ -640,6 +643,13 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func TestGetReferredSubject(t *testing.T) {
|
||||
Convey("GetReferredSubject error", t, func() {
|
||||
_, err := repodb.GetReferredSubject([]byte("bad json"))
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
}
|
||||
|
||||
func skipIt(t *testing.T) {
|
||||
t.Helper()
|
||||
|
|
@ -11,6 +11,7 @@ import (
|
|||
const (
|
||||
ManifestDataBucket = "ManifestData"
|
||||
IndexDataBucket = "IndexData"
|
||||
ArtifactDataBucket = "ArtifactData"
|
||||
UserMetadataBucket = "UserMeta"
|
||||
RepoMetadataBucket = "RepoMetadata"
|
||||
VersionBucket = "Version"
|
||||
|
@ -67,6 +68,26 @@ type RepoDB interface { //nolint:interfacebloat
|
|||
// GetIndexData returns indexData for a given Index from the database
|
||||
GetIndexData(indexDigest godigest.Digest) (IndexData, error)
|
||||
|
||||
// SetArtifactData sets artifactData for a given artifact in the database
|
||||
SetArtifactData(artifactDigest godigest.Digest, artifactData ArtifactData) error
|
||||
|
||||
// GetArtifactData returns artifactData for a given artifact from the database
|
||||
GetArtifactData(artifactDigest godigest.Digest) (ArtifactData, error)
|
||||
|
||||
// SetReferrer adds a referrer to the referrers list of a manifest inside a repo
|
||||
SetReferrer(repo string, referredDigest godigest.Digest, referrer Descriptor) error
|
||||
|
||||
// SetReferrer delets a referrer to the referrers list of a manifest inside a repo
|
||||
DeleteReferrer(repo string, referredDigest godigest.Digest, referrerDigest godigest.Digest) error
|
||||
|
||||
// GetReferrers returns the list of referrers for a referred manifest
|
||||
GetReferrers(repo string, referredDigest godigest.Digest) ([]Descriptor, error)
|
||||
|
||||
// GetFilteredReferrersInfo returnes a list of for all referrers of the given digest that match one of the
|
||||
// artifact types.
|
||||
GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string) (
|
||||
[]ReferrerInfo, error)
|
||||
|
||||
// IncrementManifestDownloads adds 1 to the download count of a manifest
|
||||
IncrementImageDownloads(repo string, reference string) error
|
||||
|
||||
|
@ -107,6 +128,18 @@ type ManifestData struct {
|
|||
ConfigBlob []byte
|
||||
}
|
||||
|
||||
type ArtifactData struct {
|
||||
ManifestBlob []byte
|
||||
}
|
||||
|
||||
type ReferrerInfo struct {
|
||||
Digest string
|
||||
MediaType string
|
||||
ArtifactType string
|
||||
Size int
|
||||
Annotations map[string]string
|
||||
}
|
||||
|
||||
// Descriptor represents an image. Multiple images might have the same digests but different tags.
|
||||
type Descriptor struct {
|
||||
Digest string
|
||||
|
@ -125,7 +158,9 @@ type RepoMetadata struct {
|
|||
|
||||
Statistics map[string]DescriptorStatistics
|
||||
Signatures map[string]ManifestSignatures
|
||||
Stars int
|
||||
Referrers map[string][]Descriptor
|
||||
|
||||
Stars int
|
||||
}
|
||||
|
||||
type LayerInfo struct {
|
||||
|
|
|
@ -76,6 +76,7 @@ func TestDynamoDBWrapper(t *testing.T) {
|
|||
manifestDataTablename := "ManifestDataTable" + uuid.String()
|
||||
versionTablename := "Version" + uuid.String()
|
||||
indexDataTablename := "IndexDataTable" + uuid.String()
|
||||
artifactDataTablename := "ArtifactDataTable" + uuid.String()
|
||||
|
||||
Convey("DynamoDB Wrapper", t, func() {
|
||||
dynamoDBDriverParams := dynamoParams.DBDriverParameters{
|
||||
|
@ -83,6 +84,7 @@ func TestDynamoDBWrapper(t *testing.T) {
|
|||
RepoMetaTablename: repoMetaTablename,
|
||||
ManifestDataTablename: manifestDataTablename,
|
||||
IndexDataTablename: indexDataTablename,
|
||||
ArtifactDataTablename: artifactDataTablename,
|
||||
VersionTablename: versionTablename,
|
||||
Region: "us-east-2",
|
||||
}
|
||||
|
@ -1757,6 +1759,202 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
|
|||
_, err = repoDB.GetIndexData(godigest.FromString("inexistent"))
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("Test artifact logic", func() {
|
||||
artifact, err := test.GetRandomArtifact(nil)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
artifactDigest, err := artifact.Digest()
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
artifactData, err := artifact.ArtifactData()
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = repoDB.SetArtifactData(artifactDigest, artifactData)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
result, err := repoDB.GetArtifactData(artifactDigest)
|
||||
So(err, ShouldBeNil)
|
||||
So(result, ShouldResemble, artifactData)
|
||||
|
||||
_, err = repoDB.GetArtifactData(godigest.FromString("inexistent"))
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("Test Referrers", func() {
|
||||
image, err := test.GetRandomImage("tag")
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
referredDigest, err := image.Digest()
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
manifestBlob, err := json.Marshal(image.Manifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
configBlob, err := json.Marshal(image.Config)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
manifestData := repodb.ManifestData{
|
||||
ManifestBlob: manifestBlob,
|
||||
ConfigBlob: configBlob,
|
||||
}
|
||||
|
||||
err = repoDB.SetManifestData(referredDigest, manifestData)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = repoDB.SetRepoReference("repo", "tag", referredDigest, ispec.MediaTypeImageManifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
// ------- Add Artifact 1
|
||||
|
||||
artifact1, err := test.GetRandomArtifact(&ispec.Descriptor{
|
||||
Digest: referredDigest,
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
artifactDigest1, err := artifact1.Digest()
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
|
||||
Digest: artifactDigest1.String(),
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
// ------- Add Artifact 2
|
||||
|
||||
artifact2, err := test.GetRandomArtifact(&ispec.Descriptor{
|
||||
Digest: referredDigest,
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
artifactDigest2, err := artifact2.Digest()
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
|
||||
Digest: artifactDigest2.String(),
|
||||
MediaType: ispec.MediaTypeArtifactManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
// ------ GetReferrers
|
||||
|
||||
referrers, err := repoDB.GetReferrers("repo", referredDigest)
|
||||
So(len(referrers), ShouldEqual, 2)
|
||||
So(referrers, ShouldContain, repodb.Descriptor{
|
||||
Digest: artifactDigest1.String(),
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
})
|
||||
So(referrers, ShouldContain, repodb.Descriptor{
|
||||
Digest: artifactDigest2.String(),
|
||||
MediaType: ispec.MediaTypeArtifactManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
// ------ DeleteReferrers
|
||||
|
||||
err = repoDB.DeleteReferrer("repo", referredDigest, artifactDigest1)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = repoDB.DeleteReferrer("repo", referredDigest, artifactDigest2)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
referrers, err = repoDB.GetReferrers("repo", referredDigest)
|
||||
So(err, ShouldBeNil)
|
||||
So(len(referrers), ShouldEqual, 0)
|
||||
})
|
||||
|
||||
Convey("Test Referrers on empty Repo", func() {
|
||||
repoMeta, err := repoDB.GetRepoMeta("repo")
|
||||
So(err, ShouldNotBeNil)
|
||||
So(repoMeta, ShouldResemble, repodb.RepoMetadata{})
|
||||
|
||||
referredDigest := godigest.FromString("referredDigest")
|
||||
referrerDigest := godigest.FromString("referrerDigest")
|
||||
|
||||
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
|
||||
Digest: referrerDigest.String(),
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
repoMeta, err = repoDB.GetRepoMeta("repo")
|
||||
So(err, ShouldBeNil)
|
||||
So(repoMeta.Referrers[referredDigest.String()][0].Digest, ShouldResemble, referrerDigest.String())
|
||||
})
|
||||
|
||||
Convey("Test Referrers add same one twice", func() {
|
||||
repoMeta, err := repoDB.GetRepoMeta("repo")
|
||||
So(err, ShouldNotBeNil)
|
||||
So(repoMeta, ShouldResemble, repodb.RepoMetadata{})
|
||||
|
||||
referredDigest := godigest.FromString("referredDigest")
|
||||
referrerDigest := godigest.FromString("referrerDigest")
|
||||
|
||||
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
|
||||
Digest: referrerDigest.String(),
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
|
||||
Digest: referrerDigest.String(),
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
repoMeta, err = repoDB.GetRepoMeta("repo")
|
||||
So(err, ShouldBeNil)
|
||||
So(len(repoMeta.Referrers[referredDigest.String()]), ShouldEqual, 1)
|
||||
})
|
||||
|
||||
Convey("GetFilteredReferrersInfo", func() {
|
||||
referredDigest := godigest.FromString("referredDigest")
|
||||
|
||||
err := repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
|
||||
Digest: "inexistendManifestDigest",
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
|
||||
Digest: "inexistendArtifactManifestDigest",
|
||||
MediaType: ispec.MediaTypeArtifactManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
// ------- Set existent manifest and artifact manifest
|
||||
err = repoDB.SetManifestData("goodManifest", repodb.ManifestData{
|
||||
ManifestBlob: []byte(`{"artifactType": "unwantedType"}`),
|
||||
ConfigBlob: []byte("{}"),
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
|
||||
Digest: "goodManifest",
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = repoDB.SetArtifactData("goodArtifact", repodb.ArtifactData{
|
||||
ManifestBlob: []byte(`{"artifactType": "wantedType"}`),
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
|
||||
Digest: "goodArtifact",
|
||||
MediaType: ispec.MediaTypeArtifactManifest,
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
referrerInfo, err := repoDB.GetFilteredReferrersInfo("repo", referredDigest, []string{"wantedType"})
|
||||
So(err, ShouldBeNil)
|
||||
So(len(referrerInfo), ShouldEqual, 1)
|
||||
So(referrerInfo[0].ArtifactType, ShouldResemble, "wantedType")
|
||||
So(referrerInfo[0].Digest, ShouldResemble, "goodArtifact")
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
@ -20,6 +20,7 @@ func TestCreateDynamo(t *testing.T) {
|
|||
RepoMetaTablename: "RepoMetadataTable",
|
||||
ManifestDataTablename: "ManifestDataTable",
|
||||
IndexDataTablename: "IndexDataTable",
|
||||
ArtifactDataTablename: "ArtifactDataTable",
|
||||
VersionTablename: "Version",
|
||||
Region: "us-east-2",
|
||||
}
|
||||
|
|
|
@ -7,19 +7,20 @@ import (
|
|||
zerr "zotregistry.io/zot/errors"
|
||||
"zotregistry.io/zot/pkg/log"
|
||||
"zotregistry.io/zot/pkg/meta/repodb"
|
||||
"zotregistry.io/zot/pkg/meta/repodb/common"
|
||||
"zotregistry.io/zot/pkg/storage"
|
||||
)
|
||||
|
||||
// OnUpdateManifest is called when a new manifest is added. It updates repodb according to the type
|
||||
// of image pushed(normal images, signatues, etc.). In care of any errors, it makes sure to keep
|
||||
// consistency between repodb and the image store.
|
||||
func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest, body []byte,
|
||||
func OnUpdateManifest(repo, reference, mediaType string, digest godigest.Digest, body []byte,
|
||||
storeController storage.StoreController, repoDB repodb.RepoDB, log log.Logger,
|
||||
) error {
|
||||
imgStore := storeController.GetImageStore(name)
|
||||
imgStore := storeController.GetImageStore(repo)
|
||||
|
||||
// check if image is a signature
|
||||
isSignature, signatureType, signedManifestDigest, err := storage.CheckIsImageSignature(name, body, reference,
|
||||
isSignature, signatureType, signedManifestDigest, err := storage.CheckIsImageSignature(repo, body, reference,
|
||||
storeController)
|
||||
if err != nil {
|
||||
if errors.Is(err, zerr.ErrOrphanSignature) {
|
||||
|
@ -30,8 +31,8 @@ func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest,
|
|||
|
||||
log.Error().Err(err).Msg("can't check if image is a signature or not")
|
||||
|
||||
if err := imgStore.DeleteImageManifest(name, reference, false); err != nil {
|
||||
log.Error().Err(err).Msgf("couldn't remove image manifest %s in repo %s", reference, name)
|
||||
if err := imgStore.DeleteImageManifest(repo, reference, false); err != nil {
|
||||
log.Error().Err(err).Msgf("couldn't remove image manifest %s in repo %s", reference, repo)
|
||||
|
||||
return err
|
||||
}
|
||||
|
@ -42,7 +43,7 @@ func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest,
|
|||
metadataSuccessfullySet := true
|
||||
|
||||
if isSignature {
|
||||
err = repoDB.AddManifestSignature(name, signedManifestDigest, repodb.SignatureMetadata{
|
||||
err = repoDB.AddManifestSignature(repo, signedManifestDigest, repodb.SignatureMetadata{
|
||||
SignatureType: signatureType,
|
||||
SignatureDigest: digest.String(),
|
||||
})
|
||||
|
@ -51,7 +52,7 @@ func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest,
|
|||
metadataSuccessfullySet = false
|
||||
}
|
||||
} else {
|
||||
err := repodb.SetMetadataFromInput(name, reference, mediaType, digest, body,
|
||||
err := repodb.SetMetadataFromInput(repo, reference, mediaType, digest, body,
|
||||
imgStore, repoDB, log)
|
||||
if err != nil {
|
||||
metadataSuccessfullySet = false
|
||||
|
@ -59,10 +60,10 @@ func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest,
|
|||
}
|
||||
|
||||
if !metadataSuccessfullySet {
|
||||
log.Info().Msgf("uploding image meta was unsuccessful for tag %s in repo %s", reference, name)
|
||||
log.Info().Msgf("uploding image meta was unsuccessful for tag %s in repo %s", reference, repo)
|
||||
|
||||
if err := imgStore.DeleteImageManifest(name, reference, false); err != nil {
|
||||
log.Error().Err(err).Msgf("couldn't remove image manifest %s in repo %s", reference, name)
|
||||
if err := imgStore.DeleteImageManifest(repo, reference, false); err != nil {
|
||||
log.Error().Err(err).Msgf("couldn't remove image manifest %s in repo %s", reference, repo)
|
||||
|
||||
return err
|
||||
}
|
||||
|
@ -76,12 +77,12 @@ func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest,
|
|||
// OnDeleteManifest is called when a manifest is deleted. It updates repodb according to the type
|
||||
// of image pushed(normal images, signatues, etc.). In care of any errors, it makes sure to keep
|
||||
// consistency between repodb and the image store.
|
||||
func OnDeleteManifest(name, reference, mediaType string, digest godigest.Digest, manifestBlob []byte,
|
||||
func OnDeleteManifest(repo, reference, mediaType string, digest godigest.Digest, manifestBlob []byte,
|
||||
storeController storage.StoreController, repoDB repodb.RepoDB, log log.Logger,
|
||||
) error {
|
||||
imgStore := storeController.GetImageStore(name)
|
||||
imgStore := storeController.GetImageStore(repo)
|
||||
|
||||
isSignature, signatureType, signedManifestDigest, err := storage.CheckIsImageSignature(name, manifestBlob,
|
||||
isSignature, signatureType, signedManifestDigest, err := storage.CheckIsImageSignature(repo, manifestBlob,
|
||||
reference, storeController)
|
||||
if err != nil {
|
||||
if errors.Is(err, zerr.ErrOrphanSignature) {
|
||||
|
@ -98,7 +99,7 @@ func OnDeleteManifest(name, reference, mediaType string, digest godigest.Digest,
|
|||
manageRepoMetaSuccessfully := true
|
||||
|
||||
if isSignature {
|
||||
err = repoDB.DeleteSignature(name, signedManifestDigest, repodb.SignatureMetadata{
|
||||
err = repoDB.DeleteSignature(repo, signedManifestDigest, repodb.SignatureMetadata{
|
||||
SignatureDigest: digest.String(),
|
||||
SignatureType: signatureType,
|
||||
})
|
||||
|
@ -107,22 +108,31 @@ func OnDeleteManifest(name, reference, mediaType string, digest godigest.Digest,
|
|||
manageRepoMetaSuccessfully = false
|
||||
}
|
||||
} else {
|
||||
err = repoDB.DeleteRepoTag(name, reference)
|
||||
err = repoDB.DeleteRepoTag(repo, reference)
|
||||
if err != nil {
|
||||
log.Info().Msg("repodb: restoring image store")
|
||||
|
||||
// restore image store
|
||||
_, err := imgStore.PutImageManifest(name, reference, mediaType, manifestBlob)
|
||||
_, err := imgStore.PutImageManifest(repo, reference, mediaType, manifestBlob)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("repodb: error while restoring image store, database is not consistent")
|
||||
}
|
||||
|
||||
manageRepoMetaSuccessfully = false
|
||||
}
|
||||
|
||||
if refferredDigest, hasSubject := common.GetReferredSubject(manifestBlob); hasSubject {
|
||||
err := repoDB.DeleteReferrer(repo, refferredDigest, digest)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("repodb: error while deleting referrer")
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !manageRepoMetaSuccessfully {
|
||||
log.Info().Msgf("repodb: deleting image meta was unsuccessful for tag %s in repo %s", reference, name)
|
||||
log.Info().Msgf("repodb: deleting image meta was unsuccessful for tag %s in repo %s", reference, repo)
|
||||
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -6,9 +6,9 @@ import (
|
|||
"testing"
|
||||
"time"
|
||||
|
||||
notreg "github.com/notaryproject/notation-go/registry"
|
||||
godigest "github.com/opencontainers/go-digest"
|
||||
ispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
oras "github.com/oras-project/artifacts-spec/specs-go/v1"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
|
||||
zerr "zotregistry.io/zot/errors"
|
||||
|
@ -95,10 +95,11 @@ func TestUpdateErrors(t *testing.T) {
|
|||
log := log.NewLogger("debug", "")
|
||||
|
||||
Convey("zerr.ErrOrphanSignature", func() {
|
||||
manifestContent := oras.Manifest{
|
||||
Subject: &oras.Descriptor{
|
||||
manifestContent := ispec.Artifact{
|
||||
Subject: &ispec.Descriptor{
|
||||
Digest: "123",
|
||||
},
|
||||
ArtifactType: notreg.ArtifactTypeNotation,
|
||||
}
|
||||
manifestBlob, err := json.Marshal(manifestContent)
|
||||
So(err, ShouldBeNil)
|
||||
|
@ -120,10 +121,11 @@ func TestUpdateErrors(t *testing.T) {
|
|||
log := log.NewLogger("debug", "")
|
||||
|
||||
Convey("CheckIsImageSignature errors", func() {
|
||||
manifestContent := oras.Manifest{
|
||||
Subject: &oras.Descriptor{
|
||||
manifestContent := ispec.Artifact{
|
||||
Subject: &ispec.Descriptor{
|
||||
Digest: "123",
|
||||
},
|
||||
ArtifactType: notreg.ArtifactTypeNotation,
|
||||
}
|
||||
manifestBlob, err := json.Marshal(manifestContent)
|
||||
So(err, ShouldBeNil)
|
||||
|
@ -143,6 +145,25 @@ func TestUpdateErrors(t *testing.T) {
|
|||
err = repoDBUpdate.OnDeleteManifest("repo", "tag1", "digest", "media", manifestBlob,
|
||||
storeController, repoDB, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
|
||||
return []byte{}, "", "", zerr.ErrManifestNotFound
|
||||
}
|
||||
|
||||
err = repoDBUpdate.OnDeleteManifest("repo", "tag1", "digest", "media", manifestBlob,
|
||||
storeController, repoDB, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("DeleteReferrers errors", func() {
|
||||
repoDB.DeleteReferrerFn = func(repo string, referredDigest, referrerDigest godigest.Digest) error {
|
||||
return ErrTestError
|
||||
}
|
||||
|
||||
err := repoDBUpdate.OnDeleteManifest("repo", "tag1", "digest", "media",
|
||||
[]byte(`{"subject": {"digest": "dig"}}`),
|
||||
storeController, repoDB, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
})
|
||||
|
||||
|
@ -153,10 +174,11 @@ func TestUpdateErrors(t *testing.T) {
|
|||
log := log.NewLogger("debug", "")
|
||||
|
||||
Convey("CheckIsImageSignature errors", func() {
|
||||
manifestContent := oras.Manifest{
|
||||
Subject: &oras.Descriptor{
|
||||
manifestContent := ispec.Artifact{
|
||||
Subject: &ispec.Descriptor{
|
||||
Digest: "123",
|
||||
},
|
||||
ArtifactType: notreg.ArtifactTypeNotation,
|
||||
}
|
||||
manifestBlob, err := json.Marshal(manifestContent)
|
||||
So(err, ShouldBeNil)
|
||||
|
@ -205,5 +227,56 @@ func TestUpdateErrors(t *testing.T) {
|
|||
manifestBlob, imageStore, repoDB, log)
|
||||
So(err, ShouldBeNil)
|
||||
})
|
||||
|
||||
Convey("SetMetadataFromInput SetData errors", func() {
|
||||
imageStore := mocks.MockedImageStore{}
|
||||
log := log.NewLogger("debug", "")
|
||||
|
||||
repoDB := mocks.RepoDBMock{
|
||||
SetManifestDataFn: func(manifestDigest godigest.Digest, mm repodb.ManifestData) error {
|
||||
return ErrTestError
|
||||
},
|
||||
}
|
||||
err := repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest",
|
||||
[]byte("{}"), imageStore, repoDB, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
repoDB = mocks.RepoDBMock{
|
||||
SetIndexDataFn: func(digest godigest.Digest, indexData repodb.IndexData) error {
|
||||
return ErrTestError
|
||||
},
|
||||
}
|
||||
err = repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeImageIndex, "digest",
|
||||
[]byte("{}"), imageStore, repoDB, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
repoDB = mocks.RepoDBMock{
|
||||
SetArtifactDataFn: func(digest godigest.Digest, artifactData repodb.ArtifactData) error {
|
||||
return ErrTestError
|
||||
},
|
||||
}
|
||||
err = repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeArtifactManifest, "digest",
|
||||
[]byte("{}"), imageStore, repoDB, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("SetMetadataFromInput SetReferrer errors", func() {
|
||||
imageStore := mocks.MockedImageStore{
|
||||
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
|
||||
return []byte("{}"), nil
|
||||
},
|
||||
}
|
||||
log := log.NewLogger("debug", "")
|
||||
|
||||
repoDB := mocks.RepoDBMock{
|
||||
SetReferrerFn: func(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error {
|
||||
return ErrTestError
|
||||
},
|
||||
}
|
||||
|
||||
err := repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest",
|
||||
[]byte(`{"subject": {"digest": "subjDigest"}}`), imageStore, repoDB, log)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
|
|
@ -119,6 +119,7 @@ func TestVersioningDynamoDB(t *testing.T) {
|
|||
Region: region,
|
||||
RepoMetaTablename: "RepoMetadataTable",
|
||||
ManifestDataTablename: "ManifestDataTable",
|
||||
ArtifactDataTablename: "ArtifactDataTable",
|
||||
IndexDataTablename: "IndexDataTable",
|
||||
VersionTablename: "Version",
|
||||
})
|
||||
|
|
|
@ -7,6 +7,7 @@ import (
|
|||
"strings"
|
||||
|
||||
"github.com/gobwas/glob"
|
||||
notreg "github.com/notaryproject/notation-go/registry"
|
||||
godigest "github.com/opencontainers/go-digest"
|
||||
imeta "github.com/opencontainers/image-spec/specs-go"
|
||||
ispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
|
@ -18,6 +19,12 @@ import (
|
|||
storageConstants "zotregistry.io/zot/pkg/storage/constants"
|
||||
)
|
||||
|
||||
func SignatureMediaTypes() map[string]bool {
|
||||
return map[string]bool{
|
||||
notreg.ArtifactTypeNotation: true,
|
||||
}
|
||||
}
|
||||
|
||||
func GetTagsByIndex(index ispec.Index) []string {
|
||||
tags := make([]string, 0)
|
||||
|
||||
|
@ -711,7 +718,7 @@ func IsSupportedMediaType(mediaType string) bool {
|
|||
mediaType == oras.MediaTypeArtifactManifest
|
||||
}
|
||||
|
||||
// imageIsSignature checks if the given image (repo:tag) represents a signature. The function
|
||||
// CheckIsImageSignature checks if the given image (repo:tag) represents a signature. The function
|
||||
// returns:
|
||||
//
|
||||
// - bool: if the image is a signature or not
|
||||
|
@ -726,7 +733,7 @@ func CheckIsImageSignature(repoName string, manifestBlob []byte, reference strin
|
|||
) (bool, string, godigest.Digest, error) {
|
||||
const cosign = "cosign"
|
||||
|
||||
var manifestContent oras.Manifest
|
||||
var manifestContent ispec.Artifact
|
||||
|
||||
err := json.Unmarshal(manifestBlob, &manifestContent)
|
||||
if err != nil {
|
||||
|
@ -734,7 +741,7 @@ func CheckIsImageSignature(repoName string, manifestBlob []byte, reference strin
|
|||
}
|
||||
|
||||
// check notation signature
|
||||
if manifestContent.Subject != nil {
|
||||
if _, ok := SignatureMediaTypes()[manifestContent.ArtifactType]; ok && manifestContent.Subject != nil {
|
||||
imgStore := storeController.GetImageStore(repoName)
|
||||
|
||||
_, signedImageManifestDigest, _, err := imgStore.GetImageManifest(repoName,
|
||||
|
|
|
@ -105,6 +105,37 @@ func (img Image) Digest() (godigest.Digest, error) {
|
|||
return godigest.FromBytes(blob), nil
|
||||
}
|
||||
|
||||
type Artifact struct {
|
||||
Manifest ispec.Artifact
|
||||
Blobs []ArtifactBlobs
|
||||
Reference string
|
||||
}
|
||||
|
||||
func (a Artifact) Digest() (godigest.Digest, error) {
|
||||
blob, err := json.Marshal(a.Manifest)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return godigest.FromBytes(blob), nil
|
||||
}
|
||||
|
||||
func (a Artifact) ArtifactData() (repodb.ArtifactData, error) {
|
||||
blob, err := json.Marshal(a.Manifest)
|
||||
if err != nil {
|
||||
return repodb.ArtifactData{}, err
|
||||
}
|
||||
|
||||
return repodb.ArtifactData{
|
||||
ManifestBlob: blob,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type ArtifactBlobs struct {
|
||||
Blob []byte
|
||||
MediaType string
|
||||
}
|
||||
|
||||
type MultiarchImage struct {
|
||||
Index ispec.Index
|
||||
Images []Image
|
||||
|
@ -623,6 +654,22 @@ func GetRandomImageComponents(layerSize int) (ispec.Image, [][]byte, ispec.Manif
|
|||
return config, layers, manifest, nil
|
||||
}
|
||||
|
||||
func GetRandomImage(reference string) (Image, error) {
|
||||
const layerSize = 20
|
||||
|
||||
config, layers, manifest, err := GetRandomImageComponents(layerSize)
|
||||
if err != nil {
|
||||
return Image{}, err
|
||||
}
|
||||
|
||||
return Image{
|
||||
Manifest: manifest,
|
||||
Layers: layers,
|
||||
Config: config,
|
||||
Reference: reference,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func GetImageComponentsWithConfig(conf ispec.Image) (ispec.Image, [][]byte, ispec.Manifest, error) {
|
||||
configBlob, err := json.Marshal(conf)
|
||||
if err = Error(err); err != nil {
|
||||
|
@ -728,6 +775,49 @@ func GetImageWithComponents(config ispec.Image, layers [][]byte) (Image, error)
|
|||
}, nil
|
||||
}
|
||||
|
||||
func GetRandomArtifact(subject *ispec.Descriptor) (Artifact, error) {
|
||||
var randBlob [10]byte
|
||||
|
||||
_, err := rand.Read(randBlob[:])
|
||||
if err != nil {
|
||||
return Artifact{}, err
|
||||
}
|
||||
|
||||
artifactBlobs := []ArtifactBlobs{
|
||||
{
|
||||
Blob: randBlob[:],
|
||||
MediaType: "application/octet-stream",
|
||||
},
|
||||
}
|
||||
|
||||
blobsDescriptors := make([]ispec.Descriptor, 0, len(artifactBlobs))
|
||||
|
||||
for _, artifactBlob := range artifactBlobs {
|
||||
blobsDescriptors = append(blobsDescriptors, ispec.Descriptor{
|
||||
Digest: godigest.FromBytes(artifactBlob.Blob),
|
||||
MediaType: artifactBlob.MediaType,
|
||||
Size: int64(len(artifactBlob.Blob)),
|
||||
})
|
||||
}
|
||||
|
||||
artifactManifest := ispec.Artifact{
|
||||
MediaType: ispec.MediaTypeArtifactManifest,
|
||||
Blobs: blobsDescriptors,
|
||||
Subject: subject,
|
||||
}
|
||||
|
||||
artifactManifestBlob, err := json.Marshal(artifactManifest)
|
||||
if err != nil {
|
||||
return Artifact{}, err
|
||||
}
|
||||
|
||||
return Artifact{
|
||||
Manifest: artifactManifest,
|
||||
Blobs: artifactBlobs,
|
||||
Reference: godigest.FromBytes(artifactManifestBlob).String(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func GetCosignSignatureTagForManifest(manifest ispec.Manifest) (string, error) {
|
||||
manifestBlob, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
|
@ -743,6 +833,32 @@ func GetCosignSignatureTagForDigest(manifestDigest godigest.Digest) string {
|
|||
return manifestDigest.Algorithm().String() + "-" + manifestDigest.Encoded() + ".sig"
|
||||
}
|
||||
|
||||
func GetImageWithSubject(subjectDigest godigest.Digest, mediaType string) (Image, error) {
|
||||
num := 100
|
||||
|
||||
conf, layers, manifest, err := GetRandomImageComponents(num)
|
||||
if err != nil {
|
||||
return Image{}, err
|
||||
}
|
||||
|
||||
manifest.Subject = &ispec.Descriptor{
|
||||
Digest: subjectDigest,
|
||||
MediaType: mediaType,
|
||||
}
|
||||
|
||||
manifestBlob, err := json.Marshal(manifest)
|
||||
if err != nil {
|
||||
return Image{}, err
|
||||
}
|
||||
|
||||
return Image{
|
||||
Manifest: manifest,
|
||||
Config: conf,
|
||||
Layers: layers,
|
||||
Reference: godigest.FromBytes(manifestBlob).String(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func UploadImage(img Image, baseURL, repo string) error {
|
||||
for _, blob := range img.Layers {
|
||||
resp, err := resty.R().Post(baseURL + "/v2/" + repo + "/blobs/uploads/")
|
||||
|
@ -830,7 +946,19 @@ func UploadImage(img Image, baseURL, repo string) error {
|
|||
return err
|
||||
}
|
||||
|
||||
func UploadArtifact(baseURL, repo string, artifactManifest *ispec.Artifact) error {
|
||||
func DeleteImage(repo, reference, baseURL string) (int, error) {
|
||||
resp, err := resty.R().Delete(
|
||||
fmt.Sprintf(baseURL+"/v2/%s/manifests/%s", repo, reference),
|
||||
)
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
|
||||
return resp.StatusCode(), err
|
||||
}
|
||||
|
||||
// UploadArtifactManifest is used in tests where we don't need to upload the blobs of the artifact.
|
||||
func UploadArtifactManifest(artifactManifest *ispec.Artifact, baseURL, repo string) error {
|
||||
// put manifest
|
||||
artifactManifestBlob, err := json.Marshal(artifactManifest)
|
||||
if err != nil {
|
||||
|
|
|
@ -8,6 +8,7 @@ import (
|
|||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"testing"
|
||||
|
@ -21,10 +22,12 @@ import (
|
|||
|
||||
"zotregistry.io/zot/pkg/api"
|
||||
"zotregistry.io/zot/pkg/api/config"
|
||||
"zotregistry.io/zot/pkg/storage"
|
||||
"zotregistry.io/zot/pkg/test"
|
||||
"zotregistry.io/zot/pkg/test/mocks"
|
||||
)
|
||||
|
||||
var ErrTestError = errors.New("test error")
|
||||
var ErrTestError = errors.New("ErrTestError")
|
||||
|
||||
func TestCopyFiles(t *testing.T) {
|
||||
Convey("sourceDir does not exist", t, func() {
|
||||
|
@ -238,7 +241,7 @@ func TestUploadArtifact(t *testing.T) {
|
|||
|
||||
artifact := ispec.Artifact{}
|
||||
|
||||
err := test.UploadArtifact(baseURL, "test", &artifact)
|
||||
err := test.UploadArtifactManifest(&artifact, baseURL, "test")
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
}
|
||||
|
@ -1274,3 +1277,62 @@ func TestGenerateNotationCerts(t *testing.T) {
|
|||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
}
|
||||
|
||||
func TestWriteImageToFileSystem(t *testing.T) {
|
||||
Convey("WriteImageToFileSystem errors", t, func() {
|
||||
err := test.WriteImageToFileSystem(test.Image{}, "repo", storage.StoreController{
|
||||
DefaultStore: mocks.MockedImageStore{
|
||||
InitRepoFn: func(name string) error {
|
||||
return ErrTestError
|
||||
},
|
||||
},
|
||||
})
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
err = test.WriteImageToFileSystem(
|
||||
test.Image{Layers: [][]byte{[]byte("testLayer")}},
|
||||
"repo",
|
||||
storage.StoreController{
|
||||
DefaultStore: mocks.MockedImageStore{
|
||||
FullBlobUploadFn: func(repo string, body io.Reader, digest godigest.Digest,
|
||||
) (string, int64, error) {
|
||||
return "", 0, ErrTestError
|
||||
},
|
||||
},
|
||||
})
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
count := 0
|
||||
err = test.WriteImageToFileSystem(
|
||||
test.Image{Layers: [][]byte{[]byte("testLayer")}},
|
||||
"repo",
|
||||
storage.StoreController{
|
||||
DefaultStore: mocks.MockedImageStore{
|
||||
FullBlobUploadFn: func(repo string, body io.Reader, digest godigest.Digest,
|
||||
) (string, int64, error) {
|
||||
if count == 0 {
|
||||
count++
|
||||
|
||||
return "", 0, nil
|
||||
}
|
||||
|
||||
return "", 0, ErrTestError
|
||||
},
|
||||
},
|
||||
})
|
||||
So(err, ShouldNotBeNil)
|
||||
|
||||
err = test.WriteImageToFileSystem(
|
||||
test.Image{Layers: [][]byte{[]byte("testLayer")}},
|
||||
"repo",
|
||||
storage.StoreController{
|
||||
DefaultStore: mocks.MockedImageStore{
|
||||
PutImageManifestFn: func(repo, reference, mediaType string, body []byte,
|
||||
) (godigest.Digest, error) {
|
||||
return "", ErrTestError
|
||||
},
|
||||
},
|
||||
})
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
}
|
||||
|
|
|
@ -40,6 +40,19 @@ type RepoDBMock struct {
|
|||
|
||||
GetIndexDataFn func(indexDigest godigest.Digest) (repodb.IndexData, error)
|
||||
|
||||
SetArtifactDataFn func(digest godigest.Digest, artifactData repodb.ArtifactData) error
|
||||
|
||||
GetArtifactDataFn func(artifactDigest godigest.Digest) (repodb.ArtifactData, error)
|
||||
|
||||
SetReferrerFn func(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error
|
||||
|
||||
DeleteReferrerFn func(repo string, referredDigest godigest.Digest, referrerDigest godigest.Digest) error
|
||||
|
||||
GetReferrersFn func(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error)
|
||||
|
||||
GetFilteredReferrersInfoFn func(repo string, referredDigest godigest.Digest, artifactTypes []string) (
|
||||
[]repodb.ReferrerInfo, error)
|
||||
|
||||
IncrementImageDownloadsFn func(repo string, reference string) error
|
||||
|
||||
AddManifestSignatureFn func(repo string, signedManifestDigest godigest.Digest, sm repodb.SignatureMetadata) error
|
||||
|
@ -103,14 +116,6 @@ func (sdm RepoDBMock) GetRepoStars(repo string) (int, error) {
|
|||
return 0, nil
|
||||
}
|
||||
|
||||
func (sdm RepoDBMock) SetRepoLogo(repo string, logoPath string) error {
|
||||
if sdm.SetRepoLogoFn != nil {
|
||||
return sdm.SetRepoLogoFn(repo, logoPath)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sdm RepoDBMock) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest,
|
||||
mediaType string,
|
||||
) error {
|
||||
|
@ -300,3 +305,55 @@ func (sdm RepoDBMock) PatchDB() error {
|
|||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sdm RepoDBMock) SetArtifactData(digest godigest.Digest, artifactData repodb.ArtifactData) error {
|
||||
if sdm.SetArtifactDataFn != nil {
|
||||
return sdm.SetArtifactDataFn(digest, artifactData)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sdm RepoDBMock) GetArtifactData(artifactDigest godigest.Digest) (repodb.ArtifactData, error) {
|
||||
if sdm.GetArtifactDataFn != nil {
|
||||
return sdm.GetArtifactDataFn(artifactDigest)
|
||||
}
|
||||
|
||||
return repodb.ArtifactData{}, nil
|
||||
}
|
||||
|
||||
func (sdm RepoDBMock) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error {
|
||||
if sdm.SetReferrerFn != nil {
|
||||
return sdm.SetReferrerFn(repo, referredDigest, referrer)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sdm RepoDBMock) DeleteReferrer(repo string, referredDigest godigest.Digest,
|
||||
referrerDigest godigest.Digest,
|
||||
) error {
|
||||
if sdm.DeleteReferrerFn != nil {
|
||||
return sdm.DeleteReferrerFn(repo, referredDigest, referrerDigest)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sdm RepoDBMock) GetReferrers(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error) {
|
||||
if sdm.GetReferrersFn != nil {
|
||||
return sdm.GetReferrersFn(repo, referredDigest)
|
||||
}
|
||||
|
||||
return []repodb.Descriptor{}, nil
|
||||
}
|
||||
|
||||
func (sdm RepoDBMock) GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest,
|
||||
artifactTypes []string,
|
||||
) ([]repodb.ReferrerInfo, error) {
|
||||
if sdm.GetFilteredReferrersInfoFn != nil {
|
||||
return sdm.GetFilteredReferrersInfoFn(repo, referredDigest, artifactTypes)
|
||||
}
|
||||
|
||||
return []repodb.ReferrerInfo{}, nil
|
||||
}
|
||||
|
|
|
@ -37,6 +37,7 @@ function setup() {
|
|||
"cacheTablename": "BlobTable",
|
||||
"repoMetaTablename": "RepoMetadataTable",
|
||||
"manifestDataTablename": "ManifestDataTable",
|
||||
"artifactDataTablename": "ArtifactDataTable",
|
||||
"indexDataTablename": "IndexDataTable",
|
||||
"versionTablename": "Version"
|
||||
}
|
||||
|
|
80
test/blackbox/helpers_referrers.bash
Normal file
80
test/blackbox/helpers_referrers.bash
Normal file
|
@ -0,0 +1,80 @@
|
|||
ROOT_DIR=$(git rev-parse --show-toplevel)
|
||||
TEST_DATA_DIR=${ROOT_DIR}/test/data/
|
||||
OS="${OS:-linux}"
|
||||
ARCH="${ARCH:-amd64}"
|
||||
ZOT_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH}
|
||||
|
||||
mkdir -p ${TEST_DATA_DIR}
|
||||
|
||||
function verify_prerequisites {
|
||||
if [ ! -f ${BATS_RUN_TMPDIR}/.firstrun ]; then
|
||||
env | grep proxy >&3
|
||||
touch ${BATS_RUN_TMPDIR}/.firstrun
|
||||
fi
|
||||
|
||||
if [ ! -f ${ZOT_PATH} ]; then
|
||||
echo "you need to build ${ZOT_PATH} before running the tests" >&3
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ ! -f ${ZOT_MINIMAL_PATH} ]; then
|
||||
echo "you need to build ${ZOT_MINIMAL_PATH} before running tests" >&3
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ ! command -v curl ] &>/dev/null; then
|
||||
echo "you need to install curl as a prerequisite to running the tests" >&3
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ ! command -v jq ] &>/dev/null; then
|
||||
echo "you need to install jq as a prerequisite to running the tests" >&3
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ ! command -v skopeo ] &>/dev/null; then
|
||||
echo "you need to install skopeo as a prerequisite to running the tests" >&3
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ ! command -v oras ] &>/dev/null; then
|
||||
echo "you need to install oras as a prerequisite to running the tests" >&3
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
function zot_serve() {
|
||||
local zot_path=${1}
|
||||
local config_file=${2}
|
||||
local pid_dir=${3}
|
||||
${zot_path} serve ${config_file} &
|
||||
echo $! >>${pid_dir}/zot.pid
|
||||
}
|
||||
|
||||
function zot_stop() {
|
||||
local pid_dir=${1}
|
||||
cat ${pid_dir}/zot.pid
|
||||
kill $(cat ${pid_dir}/zot.pid)
|
||||
rm ${pid_dir}/zot.pid
|
||||
}
|
||||
|
||||
function setup_zot_file_level() {
|
||||
local config_file=${1}
|
||||
zot_serve ${ZOT_PATH} ${config_file} ${BATS_FILE_TMPDIR}
|
||||
}
|
||||
|
||||
function teardown_zot_file_level() {
|
||||
zot_stop ${BATS_FILE_TMPDIR}
|
||||
}
|
||||
|
||||
function wait_zot_reachable() {
|
||||
zot_url=${1}
|
||||
curl --connect-timeout 3 \
|
||||
--max-time 3 \
|
||||
--retry 10 \
|
||||
--retry-delay 0 \
|
||||
--retry-max-time 60 \
|
||||
--retry-connrefused \
|
||||
${zot_url}
|
||||
}
|
109
test/blackbox/referrers.bats
Normal file
109
test/blackbox/referrers.bats
Normal file
|
@ -0,0 +1,109 @@
|
|||
load helpers_referrers
|
||||
|
||||
function setup() {
|
||||
# Verify prerequisites are available
|
||||
if ! verify_prerequisites; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Download test data to folder common for the entire suite, not just this file
|
||||
skopeo --insecure-policy copy --format=oci docker://alpine:latest oci:${TEST_DATA_DIR}alpine:latest
|
||||
|
||||
# Setup zot server
|
||||
ZOT_ROOT_DIR=${BATS_FILE_TMPDIR}/zot
|
||||
echo ${ZOT_ROOT_DIR}
|
||||
ZOT_LOG_FILE=${ZOT_ROOT_DIR}/zot-log.json
|
||||
ZOT_CONFIG_FILE=${BATS_FILE_TMPDIR}/zot_config.json
|
||||
mkdir -p ${ZOT_ROOT_DIR}
|
||||
touch ${ZOT_LOG_FILE}
|
||||
cat >${ZOT_CONFIG_FILE} <<EOF
|
||||
{
|
||||
"distSpecVersion": "1.1.0",
|
||||
"storage": {
|
||||
"rootDirectory": "${ZOT_ROOT_DIR}"
|
||||
},
|
||||
"http": {
|
||||
"address": "0.0.0.0",
|
||||
"port": "8080"
|
||||
},
|
||||
"log": {
|
||||
"level": "debug",
|
||||
"output": "${ZOT_LOG_FILE}"
|
||||
},
|
||||
"extensions": {
|
||||
"search": {
|
||||
"enable": true
|
||||
}
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
# Add artifact contents to files
|
||||
ARTIFACT_BLOBS_DIR=${BATS_FILE_TMPDIR}/artifact-blobs
|
||||
mkdir -p ${ARTIFACT_BLOBS_DIR}
|
||||
|
||||
IMAGE_MANIFEST_REFERRER=${ARTIFACT_BLOBS_DIR}/image-manifest-ref-blob
|
||||
echo IMAGE_MANIFEST_REFERRER=${IMAGE_MANIFEST_REFERRER}
|
||||
touch ${IMAGE_MANIFEST_REFERRER}
|
||||
cat >${IMAGE_MANIFEST_REFERRER} <<EOF
|
||||
This artifact is represented as an ispec image manifest, this is the layer inside the manifest.
|
||||
EOF
|
||||
|
||||
ARTIFACT_MANIFEST_REFERRER=${ARTIFACT_BLOBS_DIR}/artifact-manifest-ref-blob
|
||||
touch ${ARTIFACT_MANIFEST_REFERRER}
|
||||
cat >${ARTIFACT_MANIFEST_REFERRER} <<EOF
|
||||
This artifact is represented as an ispec artifact manifest, this is the blob inside the manifest.
|
||||
EOF
|
||||
|
||||
setup_zot_file_level ${ZOT_CONFIG_FILE}
|
||||
echo "yes"
|
||||
wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
|
||||
|
||||
run skopeo --insecure-policy copy --dest-tls-verify=false \
|
||||
oci:${TEST_DATA_DIR}/alpine:latest \
|
||||
docker://127.0.0.1:8080/alpine:latest
|
||||
[ "$status" -eq 0 ]
|
||||
run curl http://127.0.0.1:8080/v2/_catalog
|
||||
[ "$status" -eq 0 ]
|
||||
[ $(echo "${lines[-1]}" | jq '.repositories[]') = '"alpine"' ]
|
||||
|
||||
run oras attach --plain-http --image-spec v1.1-image --artifact-type image.type 127.0.0.1:8080/alpine:latest ${IMAGE_MANIFEST_REFERRER}
|
||||
[ "$status" -eq 0 ]
|
||||
|
||||
run oras attach --plain-http --image-spec v1.1-artifact --artifact-type artifact.type 127.0.0.1:8080/alpine:latest ${ARTIFACT_MANIFEST_REFERRER}
|
||||
[ "$status" -eq 0 ]
|
||||
|
||||
MANIFEST_DIGEST=$(skopeo inspect --tls-verify=false docker://localhost:8080/alpine:latest | jq -r '.Digest')
|
||||
echo ${MANIFEST_DIGEST}
|
||||
|
||||
curl -X GET http://127.0.0.1:8080/v2/alpine/referrers/${MANIFEST_DIGEST}?artifactType=image.type
|
||||
}
|
||||
|
||||
function teardown() {
|
||||
local ZOT_ROOT_DIR=${BATS_FILE_TMPDIR}/zot
|
||||
zot_stop ${BATS_FILE_TMPDIR}
|
||||
rm -rf ${ZOT_ROOT_DIR}
|
||||
}
|
||||
|
||||
@test "add referrers, one artifact and one image" {
|
||||
|
||||
# Check referrers API using the normal REST endpoint
|
||||
run curl -X GET http://127.0.0.1:8080/v2/alpine/referrers/${MANIFEST_DIGEST}?artifactType=image.type
|
||||
[ "$status" -eq 0 ]
|
||||
[ $(echo "${lines[-1]}" | jq '.manifests[].artifactType') = '"image.type"' ]
|
||||
|
||||
run curl -X GET http://127.0.0.1:8080/v2/alpine/referrers/${MANIFEST_DIGEST}?artifactType=artifact.type
|
||||
[ "$status" -eq 0 ]
|
||||
[ $(echo "${lines[-1]}" | jq '.manifests[].artifactType') = '"artifact.type"' ]
|
||||
|
||||
# Check referrers API using the GQL endpoint
|
||||
REFERRER_QUERY_DATA="{ \"query\": \"{ Referrers(repo:\\\"alpine\\\", digest:\\\"${MANIFEST_DIGEST}\\\", type:[\\\"image.type\\\"]) { MediaType ArtifactType Digest Size} }\"}"
|
||||
run curl -X POST -H "Content-Type: application/json" --data "${REFERRER_QUERY_DATA}" http://localhost:8080/v2/_zot/ext/search
|
||||
[ "$status" -eq 0 ]
|
||||
[ $(echo "${lines[-1]}" | jq '.data.Referrers[].ArtifactType') = '"image.type"' ]
|
||||
|
||||
REFERRER_QUERY_DATA="{ \"query\": \"{ Referrers(repo:\\\"alpine\\\", digest:\\\"${MANIFEST_DIGEST}\\\", type:[\\\"artifact.type\\\"]) { MediaType ArtifactType Digest Size} }\"}"
|
||||
run curl -X POST -H "Content-Type: application/json" --data "${REFERRER_QUERY_DATA}" http://localhost:8080/v2/_zot/ext/search
|
||||
[ "$status" -eq 0 ]
|
||||
[ $(echo "${lines[-1]}" | jq '.data.Referrers[].ArtifactType') = '"artifact.type"' ]
|
||||
}
|
Loading…
Reference in a new issue