0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-30 22:34:13 -05:00

refactor: Reduce binary size of zot-minimal; Added CI check for binary size (#1758)

Signed-off-by: Alexei Dodon <adodon@cisco.com>
This commit is contained in:
Alexei Dodon 2023-09-06 19:58:00 +03:00 committed by GitHub
parent 75a76005b4
commit f5b63963be
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 134 additions and 83 deletions

View file

@ -12,10 +12,6 @@ permissions: read-all
jobs:
branch-cov:
runs-on: ubuntu-latest
strategy:
matrix:
os: [linux]
arch: [amd64]
name: coverage
steps:
- uses: actions/checkout@v4
@ -42,6 +38,4 @@ jobs:
gobco -test '-tags=sync,search,scrub,metrics,containers_image_openpgp' $i;
gobco -test '-tags=minimal,containers_image_openpgp' $i;
done
env:
OS: ${{ matrix.os }}
ARCH: ${{ matrix.arch }}

View file

@ -113,10 +113,10 @@ jobs:
echo "Building for $OS:$ARCH"
cd $GITHUB_WORKSPACE
if [[ $OS == "linux" && $ARCH == "amd64" ]]; then
make OS=$OS ARCH=$ARCH
make
sudo env "PATH=$PATH" make privileged-test
else
make OS=$OS ARCH=$ARCH binary binary-minimal binary-debug cli bench exporter-minimal
make binary binary-minimal binary-debug cli bench exporter-minimal
fi
env:
S3MOCK_ENDPOINT: localhost:4566

View file

@ -4,7 +4,6 @@ on:
tags:
- v*
branches:
- master
- main
pull_request:
permissions:

View file

@ -12,10 +12,6 @@ permissions: read-all
jobs:
tls-check:
runs-on: ubuntu-latest
strategy:
matrix:
os: [linux]
arch: [amd64]
name: TLS check
steps:
- uses: actions/checkout@v4
@ -36,8 +32,8 @@ jobs:
continue-on-error: true
run: |
cd $GITHUB_WORKSPACE
make OS=$OS ARCH=$ARCH binary
bin/zot-$OS-$ARCH serve examples/config-tls.json &
make binary
bin/zot-linux-amd64 serve examples/config-tls.json &
sleep 5
curl -kv --tls-max 1.0 -0 https://localhost:8080/v2/
if [[ "$?" -eq 0 ]]; then echo "TLSv1.0 detected"; exit 1; fi
@ -45,6 +41,3 @@ jobs:
if [[ "$?" -eq 0 ]]; then echo "TLSv1.1 detected"; exit 1; fi
curl -kv --tls-max 1.2 -0 https://localhost:8080/v2/
if [[ "$?" -ne 0 ]]; then echo "TLSv1.2 missing"; exit 1; fi
env:
OS: ${{ matrix.os }}
ARCH: ${{ matrix.arch }}

40
.github/workflows/zot-minimal-size.yml vendored Normal file
View file

@ -0,0 +1,40 @@
name: "zot minimal binary size"
on:
pull_request:
branches: [main]
permissions: read-all
jobs:
zot-minimal-size:
runs-on: ubuntu-latest
name: compare-binary-size
steps:
- uses: actions/checkout@v3
- uses: actions/setup-go@v4
with:
cache: false
go-version: 1.20.x
- name: Check if zot-minimal binary increased with more than 1%
run: |
echo "Building zot-minimal and check size"
cd $GITHUB_WORKSPACE
make binary-minimal
BINSIZE=$(stat -c%s "bin/zot-linux-amd64-minimal")
echo "Building zot-minimal on main branch and check size"
mkdir -p zot_main
git clone https://github.com/project-zot/zot zot_main/
cd zot_main
make binary-minimal
BINSIZE_MAIN=$(stat -c%s "bin/zot-linux-amd64-minimal")
cd $GITHUB_WORKSPACE && rm -rf zot_main
[[ $BINSIZE -gt $BINSIZE_MAIN ]] || exit 0
echo "PR changes increased size of zot-minimal binary"
echo "PR binary size: $BINSIZE Bytes"
echo "main branch binary size: $BINSIZE_MAIN Bytes"
PERCENTACE=$(echo "scale=2; (($BINSIZE-$BINSIZE_MAIN)*100)/$BINSIZE_MAIN" | bc)
if ((`bc <<< "$PERCENTACE>=1.0"`)); then echo "zot minimal binary increased by $PERCENTACE% comparing with main"; exit 1; fi

3
go.mod
View file

@ -15,7 +15,6 @@ require (
github.com/docker/distribution v2.8.2+incompatible
github.com/dustin/go-humanize v1.0.1
github.com/fsnotify/fsnotify v1.6.0
github.com/getlantern/deepcopy v0.0.0-20160317154340-7f45deb8130a
github.com/go-ldap/ldap/v3 v3.4.5
github.com/gofrs/uuid v4.4.0+incompatible
github.com/google/go-containerregistry v0.16.1
@ -50,7 +49,6 @@ require (
github.com/aquasecurity/trivy v0.44.1
github.com/aws/aws-sdk-go-v2/service/dynamodb v1.21.5
github.com/containers/image/v5 v5.27.0
github.com/gobwas/glob v0.2.3
github.com/google/go-github/v52 v52.0.0
github.com/gorilla/handlers v1.5.1
github.com/gorilla/securecookie v1.1.1
@ -121,6 +119,7 @@ require (
github.com/go-gorp/gorp/v3 v3.0.5 // indirect
github.com/go-jose/go-jose/v3 v3.0.0 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/google/gnostic v0.5.7-v3refs // indirect
github.com/google/go-github/v53 v53.0.0 // indirect
github.com/google/licenseclassifier/v2 v2.0.0 // indirect

2
go.sum
View file

@ -698,8 +698,6 @@ github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbS
github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88=
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
github.com/getlantern/deepcopy v0.0.0-20160317154340-7f45deb8130a h1:yU/FENpkHYISWsQrbr3pcZOBj0EuRjPzNc1+dTCLu44=
github.com/getlantern/deepcopy v0.0.0-20160317154340-7f45deb8130a/go.mod h1:AEugkNu3BjBxyz958nJ5holD9PRjta6iprcoUauDbU4=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/glebarez/go-sqlite v1.20.3 h1:89BkqGOXR9oRmG58ZrzgoY/Fhy5x0M+/WV48U5zVrZ4=

View file

@ -1,10 +1,10 @@
package config
import (
"encoding/json"
"os"
"time"
"github.com/getlantern/deepcopy"
distspec "github.com/opencontainers/distribution-spec/specs-go"
extconf "zotregistry.io/zot/pkg/extensions/config"
@ -221,17 +221,28 @@ func SameFile(str1, str2 string) (bool, error) {
return os.SameFile(sFile, tFile), nil
}
func DeepCopy(src, dst interface{}) error {
bytes, err := json.Marshal(src)
if err != nil {
return err
}
err = json.Unmarshal(bytes, dst)
return err
}
// Sanitize makes a sanitized copy of the config removing any secrets.
func (c *Config) Sanitize() *Config {
sanitizedConfig := &Config{}
if err := deepcopy.Copy(sanitizedConfig, c); err != nil {
if err := DeepCopy(c, sanitizedConfig); err != nil {
panic(err)
}
if c.HTTP.Auth != nil && c.HTTP.Auth.LDAP != nil && c.HTTP.Auth.LDAP.BindPassword != "" {
sanitizedConfig.HTTP.Auth.LDAP = &LDAPConfig{}
if err := deepcopy.Copy(sanitizedConfig.HTTP.Auth.LDAP, c.HTTP.Auth.LDAP); err != nil {
if err := DeepCopy(c.HTTP.Auth.LDAP, sanitizedConfig.HTTP.Auth.LDAP); err != nil {
panic(err)
}

View file

@ -65,4 +65,20 @@ func TestConfig(t *testing.T) {
So(err, ShouldBeNil)
So(isSame, ShouldBeTrue)
})
Convey("Test DeepCopy() & Sanitize()", t, func() {
conf := config.New()
So(conf, ShouldNotBeNil)
authConfig := &config.AuthConfig{LDAP: &config.LDAPConfig{BindPassword: "oina"}}
conf.HTTP.Auth = authConfig
So(func() { conf.Sanitize() }, ShouldNotPanic)
conf = conf.Sanitize()
So(conf.HTTP.Auth.LDAP.BindPassword, ShouldEqual, "******")
// negative
obj := make(chan int)
err := config.DeepCopy(conf, obj)
So(err, ShouldNotBeNil)
err = config.DeepCopy(obj, conf)
So(err, ShouldNotBeNil)
})
}

View file

@ -16,7 +16,6 @@ import (
"sync"
"time"
notreg "github.com/notaryproject/notation-go/registry"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sigstore/cosign/v2/pkg/oci/remote"
@ -482,7 +481,7 @@ func isNotationSigned(ctx context.Context, repo, digestStr string, searchConf se
var referrers ispec.Index
URL := fmt.Sprintf("%s/v2/%s/referrers/%s?artifactType=%s",
*searchConf.servURL, repo, digestStr, notreg.ArtifactTypeNotation)
*searchConf.servURL, repo, digestStr, common.ArtifactTypeNotation)
_, err := makeGETRequest(ctx, URL, username, password,
*searchConf.verifyTLS, *searchConf.debug, &referrers, searchConf.resultWriter)

View file

@ -23,6 +23,9 @@ const (
CosignSignature = "cosign"
CosignSigKey = "dev.cosignproject.cosign/signature"
NotationSignature = "notation"
// same value as github.com/notaryproject/notation-go/registry.ArtifactTypeNotation (assert by internal test).
// reason used: to reduce zot minimal binary size (otherwise adds oras.land/oras-go/v2 deps).
ArtifactTypeNotation = "application/vnd.cncf.notary.signature"
)
func Contains[T comparable](elems []T, v T) bool {

View file

@ -5,6 +5,7 @@ import (
"path"
"testing"
notreg "github.com/notaryproject/notation-go/registry"
. "github.com/smartystreets/goconvey/convey"
"zotregistry.io/zot/pkg/api/config"
@ -56,4 +57,8 @@ func TestCommon(t *testing.T) {
So(common.Index([]string{"a", "b"}, "b"), ShouldEqual, 1)
So(common.Index([]string{"a", "b"}, "c"), ShouldEqual, -1)
})
Convey("Test ArtifactTypeNotation const has same value as in notaryproject", t, func() {
So(common.ArtifactTypeNotation, ShouldEqual, notreg.ArtifactTypeNotation)
})
}

View file

@ -12,13 +12,11 @@ import (
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/log"
_ "zotregistry.io/zot/swagger"
)
func SetupSwaggerRoutes(conf *config.Config, router *mux.Router, authFunc mux.MiddlewareFunc,
log log.Logger,
) {
// swagger "/swagger/v2/index.html"
log.Warn().Msg("skipping enabling swagger because given zot binary " +
"doesn't include this feature, please build a binary that does so")
}

View file

@ -13,13 +13,13 @@ import (
"net/url"
"os"
"path"
"regexp"
"strconv"
"strings"
"testing"
"time"
dbTypes "github.com/aquasecurity/trivy-db/pkg/types"
"github.com/gobwas/glob"
regTypes "github.com/google/go-containerregistry/pkg/v1/types"
notreg "github.com/notaryproject/notation-go/registry"
godigest "github.com/opencontainers/go-digest"
@ -1245,9 +1245,9 @@ func TestExpandedRepoInfo(t *testing.T) {
ExpandedRepoInfo(repo:"test1"){
Summary {
Name LastUpdated Size
Platforms {Os Arch}
Platforms {Os Arch}
Vendors
}
}
Images {
Tag
Manifests {
@ -1341,7 +1341,7 @@ func TestExpandedRepoInfo(t *testing.T) {
query := `{
ExpandedRepoInfo(repo:"zot-cve-test"){
Summary {
Name LastUpdated Size
Name LastUpdated Size
}
}
}`
@ -1363,7 +1363,7 @@ func TestExpandedRepoInfo(t *testing.T) {
Images {
Tag
Manifests {
Digest
Digest
Layers {Size Digest}
}
IsSigned
@ -1437,7 +1437,7 @@ func TestExpandedRepoInfo(t *testing.T) {
ExpandedRepoInfo(repo:"zot-test"){
Images {
RepoName
Tag IsSigned
Tag IsSigned
Manifests{
Digest
Layers {Size Digest}
@ -1548,10 +1548,10 @@ func TestExpandedRepoInfo(t *testing.T) {
{
ExpandedRepoInfo(repo:"repo"){
Images {
RepoName
Tag
RepoName
Tag
Manifests {
Digest
Digest
Layers {Size Digest}
}
}
@ -1633,10 +1633,10 @@ func TestExpandedRepoInfo(t *testing.T) {
{
ExpandedRepoInfo(repo:"test-repo"){
Images {
RepoName
Tag
RepoName
Tag
Manifests {
Digest
Digest
Layers {Size Digest}
}
}
@ -1743,10 +1743,10 @@ func TestExpandedRepoInfo(t *testing.T) {
{
ExpandedRepoInfo(repo:"repo"){
Images {
RepoName
Tag
RepoName
Tag
Manifests {
Digest
Digest
Layers {Size Digest}
}
}
@ -3306,7 +3306,7 @@ func TestGlobalSearch(t *testing.T) {
Images {
RepoName Tag LastUpdated Size
Manifests {
LastUpdated Size
LastUpdated Size
Platform { Os Arch }
History {
Layer { Size Digest }
@ -3322,7 +3322,7 @@ func TestGlobalSearch(t *testing.T) {
NewestImage {
RepoName Tag LastUpdated Size
Manifests {
LastUpdated Size
LastUpdated Size
Platform { Os Arch }
History {
Layer { Size Digest }
@ -3535,7 +3535,7 @@ func TestGlobalSearch(t *testing.T) {
Images {
RepoName Tag LastUpdated Size
Manifests {
LastUpdated Size
LastUpdated Size
Platform { Os Arch }
History {
Layer { Size Digest }
@ -3551,7 +3551,7 @@ func TestGlobalSearch(t *testing.T) {
NewestImage {
RepoName Tag LastUpdated Size
Manifests {
LastUpdated Size
LastUpdated Size
Platform { Os Arch }
History {
Layer { Size Digest }
@ -3625,7 +3625,7 @@ func TestGlobalSearch(t *testing.T) {
Images {
RepoName Tag LastUpdated Size
Manifests {
LastUpdated Size
LastUpdated Size
Platform { Os Arch }
History {
Layer { Size Digest }
@ -3641,7 +3641,7 @@ func TestGlobalSearch(t *testing.T) {
NewestImage {
RepoName Tag LastUpdated Size
Manifests {
LastUpdated Size
LastUpdated Size
Platform { Os Arch }
History {
Layer { Size Digest }
@ -4335,7 +4335,7 @@ func TestMetaDBWhenSigningImages(t *testing.T) {
Images {
RepoName Tag LastUpdated Size IsSigned
Manifests{
LastUpdated Size
LastUpdated Size
}
}
}
@ -5349,7 +5349,7 @@ func TestMetaDBWhenDeletingImages(t *testing.T) {
RepoName Tag LastUpdated Size IsSigned
Manifests{
Platform { Os Arch }
LastUpdated Size
LastUpdated Size
}
}
}
@ -5399,7 +5399,7 @@ func TestMetaDBWhenDeletingImages(t *testing.T) {
RepoName Tag LastUpdated Size IsSigned
Manifests{
Platform { Os Arch }
LastUpdated Size
LastUpdated Size
}
}
}
@ -5436,9 +5436,9 @@ func TestMetaDBWhenDeletingImages(t *testing.T) {
for _, manifest := range indexContent.Manifests {
tag := manifest.Annotations[ispec.AnnotationRefName]
cosignTagRule := glob.MustCompile("sha256-*.sig")
cosignTagRule := regexp.MustCompile(`sha256\-.+\.sig`)
if cosignTagRule.Match(tag) {
if cosignTagRule.MatchString(tag) {
signatureTag = tag
}
}
@ -5472,10 +5472,10 @@ func TestMetaDBWhenDeletingImages(t *testing.T) {
{
GlobalSearch(query:"repo1:1.0.1"){
Images {
RepoName Tag LastUpdated Size IsSigned
RepoName Tag LastUpdated Size IsSigned
Manifests{
Platform { Os Arch }
LastUpdated Size
LastUpdated Size
}
}
}
@ -5786,15 +5786,15 @@ func TestSearchSize(t *testing.T) {
query := `
{
GlobalSearch(query:"testrepo:"){
Images {
Images {
RepoName Tag LastUpdated Size Vendor
Manifests{
Platform { Os Arch }
LastUpdated Size
LastUpdated Size
}
}
Repos {
Name LastUpdated Size
Name LastUpdated Size
NewestImage {
Manifests{
Platform { Os Arch }
@ -5823,15 +5823,15 @@ func TestSearchSize(t *testing.T) {
query = `
{
GlobalSearch(query:"testrepo"){
Images {
RepoName Tag LastUpdated Size
Images {
RepoName Tag LastUpdated Size
Manifests{
Platform { Os Arch }
LastUpdated Size
}
}
Repos {
Name LastUpdated Size
Name LastUpdated Size
NewestImage {
Manifests{
Platform { Os Arch }
@ -5869,19 +5869,19 @@ func TestSearchSize(t *testing.T) {
query = `
{
GlobalSearch(query:"testrepo:"){
Images {
RepoName Tag LastUpdated Size
Images {
RepoName Tag LastUpdated Size
Manifests{
Platform { Os Arch }
LastUpdated Size
LastUpdated Size
}
}
Repos {
Name LastUpdated Size
Name LastUpdated Size
NewestImage {
Manifests{
Platform { Os Arch }
LastUpdated Size
LastUpdated Size
}
}
}
@ -5905,14 +5905,14 @@ func TestSearchSize(t *testing.T) {
{
GlobalSearch(query:"testrepo"){
Images {
RepoName Tag LastUpdated Size
RepoName Tag LastUpdated Size
Manifests{
Platform { Os Arch }
LastUpdated Size
}
}
}
Repos {
Name LastUpdated Size
Name LastUpdated Size
NewestImage {
Manifests{
Platform { Os Arch }

View file

@ -9,7 +9,6 @@ import (
"fmt"
"net/http"
notreg "github.com/notaryproject/notation-go/registry"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
@ -209,7 +208,7 @@ func getNotationManifestsFromOCIRefs(ociRefs ispec.Index) []ispec.Descriptor {
notaryManifests := []ispec.Descriptor{}
for _, ref := range ociRefs.Manifests {
if ref.ArtifactType == notreg.ArtifactTypeNotation {
if ref.ArtifactType == common.ArtifactTypeNotation {
notaryManifests = append(notaryManifests, ref)
}
}

View file

@ -12,7 +12,6 @@ import (
"time"
"github.com/docker/distribution/registry/storage/driver"
notreg "github.com/notaryproject/notation-go/registry"
godigest "github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/schema"
imeta "github.com/opencontainers/image-spec/specs-go"
@ -714,7 +713,7 @@ func IsSignature(descriptor ispec.Descriptor) bool {
}
// is notation signature
if descriptor.ArtifactType == notreg.ArtifactTypeNotation {
if descriptor.ArtifactType == zcommon.ArtifactTypeNotation {
return true
}
default:

View file

@ -3,11 +3,10 @@ package storage
import (
"encoding/json"
"fmt"
"regexp"
"strings"
"github.com/docker/distribution/registry/storage/driver/factory"
"github.com/gobwas/glob"
notreg "github.com/notaryproject/notation-go/registry"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
@ -232,14 +231,14 @@ func CheckIsImageSignature(repoName string, manifestBlob []byte, reference strin
manifestArtifactType := zcommon.GetManifestArtifactType(manifestContent)
// check notation signature
if manifestArtifactType == notreg.ArtifactTypeNotation && manifestContent.Subject != nil {
if manifestArtifactType == zcommon.ArtifactTypeNotation && manifestContent.Subject != nil {
return true, NotationType, manifestContent.Subject.Digest, nil
}
// check cosign
cosignTagRule := glob.MustCompile("sha256-*.sig")
cosignTagRule := regexp.MustCompile(`sha256\-.+\.sig`)
if tag := reference; cosignTagRule.Match(reference) {
if tag := reference; cosignTagRule.MatchString(reference) {
prefixLen := len("sha256-")
digestLen := 64
signedImageManifestDigestEncoded := tag[prefixLen : prefixLen+digestLen]

View file

@ -13,7 +13,6 @@ import (
"strings"
"time"
notreg "github.com/notaryproject/notation-go/registry"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
@ -233,7 +232,7 @@ func (olu BaseOciLayoutUtils) GetImageTagsWithTimestamp(repo string) ([]cvemodel
// check notary signature corresponding to repo name, manifest digest and mediatype.
func (olu BaseOciLayoutUtils) checkNotarySignature(name string, digest godigest.Digest) bool {
imageStore := olu.StoreController.GetImageStore(name)
mediaType := notreg.ArtifactTypeNotation
mediaType := common.ArtifactTypeNotation
referrers, err := imageStore.GetReferrers(name, digest, []string{mediaType})
if err != nil {