diff --git a/.bazel/golangcilint.yaml b/.bazel/golangcilint.yaml index 2b2181d4..1775a4a0 100644 --- a/.bazel/golangcilint.yaml +++ b/.bazel/golangcilint.yaml @@ -5,6 +5,7 @@ run: linters: enable-all: true + disable: funlen,godox,gocognit output: format: colored-line-number diff --git a/Makefile b/Makefile index a46d3840..0169b688 100644 --- a/Makefile +++ b/Makefile @@ -23,8 +23,8 @@ test: .PHONY: check check: .bazel/golangcilint.yaml - golangci-lint --version || curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.17.1 - golangci-lint --config .bazel/golangcilint.yaml run --enable-all ./cmd/... ./pkg/... + golangci-lint --version || curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.21.0 + golangci-lint --config .bazel/golangcilint.yaml run --enable-all ./cmd/... ./pkg/... docs/docs.go: swag -v || go install github.com/swaggo/swag/cmd/swag diff --git a/pkg/api/config.go b/pkg/api/config.go index a4a3d1f9..6ba81228 100644 --- a/pkg/api/config.go +++ b/pkg/api/config.go @@ -82,13 +82,18 @@ func (c *Config) Sanitize() *Config { if err := deepcopy.Copy(s, c); err != nil { panic(err) } + s.HTTP.Auth.LDAP = &LDAPConfig{} + if err := deepcopy.Copy(s.HTTP.Auth.LDAP, c.HTTP.Auth.LDAP); err != nil { panic(err) } + s.HTTP.Auth.LDAP.BindPassword = "******" + return s } + return c } @@ -101,5 +106,6 @@ func (c *Config) Validate(log log.Logger) error { return errors.ErrLDAPConfig } } + return nil } diff --git a/pkg/api/controller.go b/pkg/api/controller.go index 4e81cb9e..3a679705 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -40,6 +40,7 @@ func (c *Controller) Run() error { engine := mux.NewRouter() engine.Use(log.SessionLogger(c.Log), handlers.RecoveryHandler(handlers.RecoveryLogger(c.Log), handlers.PrintRecoveryStack(false))) + c.Router = engine _ = NewRouteHandler(c) @@ -66,10 +67,13 @@ func (c *Controller) Run() error { if err != nil { panic(err) } + caCertPool := x509.NewCertPool() + if !caCertPool.AppendCertsFromPEM(caCert) { panic(errors.ErrBadCACert) } + server.TLSConfig = &tls.Config{ ClientAuth: clientAuth, ClientCAs: caCertPool, @@ -81,5 +85,6 @@ func (c *Controller) Run() error { return server.ServeTLS(l, c.Config.HTTP.TLS.Cert, c.Config.HTTP.TLS.Key) } + return server.Serve(l) } diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index 9dda7703..637af2fc 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -637,21 +637,25 @@ func newTestLDAPServer() *testLDAPServer { server.SearchFunc("", l) l.server = server l.quitCh = quitCh + return l } func (l *testLDAPServer) Start() { addr := fmt.Sprintf("%s:%d", LDAPAddress, LDAPPort) + go func() { if err := l.server.ListenAndServe(addr); err != nil { panic(err) } }() + for { _, err := net.Dial("tcp", addr) if err == nil { break } + time.Sleep(10 * time.Millisecond) } } @@ -664,10 +668,12 @@ func (l *testLDAPServer) Bind(bindDN, bindSimplePw string, conn net.Conn) (vldap if bindDN == "" || bindSimplePw == "" { return vldap.LDAPResultInappropriateAuthentication, errors.New("ldap: bind creds required") } + if (bindDN == LDAPBindDN && bindSimplePw == LDAPBindPassword) || (bindDN == fmt.Sprintf("cn=%s,%s", username, LDAPBaseDN) && bindSimplePw == passphrase) { return vldap.LDAPResultSuccess, nil } + return vldap.LDAPResultInvalidCredentials, errors.New("ldap: invalid credentials") } @@ -682,6 +688,7 @@ func (l *testLDAPServer) Search(boundDN string, req vldap.SearchRequest, ResultCode: vldap.LDAPResultSuccess, }, nil } + return vldap.ServerSearchResult{}, nil } diff --git a/pkg/api/errors.go b/pkg/api/errors.go index 63a9d7d4..f1e0b482 100644 --- a/pkg/api/errors.go +++ b/pkg/api/errors.go @@ -35,7 +35,6 @@ const ( ) func NewError(code ErrorCode, detail ...interface{}) Error { - var errMap = map[ErrorCode]Error{ BLOB_UNKNOWN: { Message: "blob unknown to registry", @@ -138,5 +137,6 @@ func NewError(code ErrorCode, detail ...interface{}) Error { e.Code = code e.Detail = detail + return e } diff --git a/pkg/api/ldap.go b/pkg/api/ldap.go index 2503fe0e..f18021c5 100644 --- a/pkg/api/ldap.go +++ b/pkg/api/ldap.go @@ -40,8 +40,11 @@ type LDAPClient struct { func (lc *LDAPClient) Connect() error { if lc.Conn == nil { var l *goldap.Conn + var err error + address := fmt.Sprintf("%s:%d", lc.Host, lc.Port) + if !lc.UseSSL { l, err = goldap.Dial("tcp", address) if err != nil { @@ -59,7 +62,9 @@ func (lc *LDAPClient) Connect() error { config.Certificates = lc.ClientCertificates config.BuildNameToCertificate() } + err = l.StartTLS(config) + if err != nil { lc.Log.Error().Err(err).Str("address", address).Msg("TLS connection failed") return err @@ -84,6 +89,7 @@ func (lc *LDAPClient) Connect() error { lc.Conn = l } + return nil } @@ -101,10 +107,12 @@ func sleepAndRetry(retries, maxRetries int) bool { if retries > maxRetries { return false } + if retries < maxRetries { time.Sleep(time.Duration(retries) * time.Second) // gradually backoff return true } + return false } @@ -130,9 +138,11 @@ func (lc *LDAPClient) Authenticate(username, password string) (bool, map[string] // clean up the cached conn, so we can retry lc.Conn.Close() lc.Conn = nil + continue } } + connected = true } @@ -144,6 +154,7 @@ func (lc *LDAPClient) Authenticate(username, password string) (bool, map[string] attributes := append(lc.Attributes, "dn") searchScope := goldap.ScopeSingleLevel + if lc.SubtreeSearch { searchScope = goldap.ScopeWholeSubtree } @@ -161,6 +172,7 @@ func (lc *LDAPClient) Authenticate(username, password string) (bool, map[string] fmt.Printf("%v\n", err) lc.Log.Error().Err(err).Str("bindDN", lc.BindDN).Str("username", username). Str("baseDN", lc.Base).Msg("search failed") + return false, nil, err } @@ -168,6 +180,7 @@ func (lc *LDAPClient) Authenticate(username, password string) (bool, map[string] err := errors.ErrBadUser lc.Log.Error().Err(err).Str("bindDN", lc.BindDN).Str("username", username). Str("baseDN", lc.Base).Msg("entries not found") + return false, nil, err } @@ -175,11 +188,13 @@ func (lc *LDAPClient) Authenticate(username, password string) (bool, map[string] err := errors.ErrEntriesExceeded lc.Log.Error().Err(err).Str("bindDN", lc.BindDN).Str("username", username). Str("baseDN", lc.Base).Msg("too many entries") + return false, nil, err } userDN := sr.Entries[0].DN user := map[string]string{} + for _, attr := range lc.Attributes { user[attr] = sr.Entries[0].GetAttributeValue(attr) } @@ -217,12 +232,16 @@ func (lc *LDAPClient) GetGroupsOfUser(username string) ([]string, error) { nil, ) sr, err := lc.Conn.Search(searchRequest) + if err != nil { return nil, err } + groups := []string{} + for _, entry := range sr.Entries { groups = append(groups, entry.GetAttributeValue("cn")) } + return groups, nil } diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 085e3849..bc1d9054 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -43,6 +43,7 @@ type RouteHandler struct { func NewRouteHandler(c *Controller) *RouteHandler { rh := &RouteHandler{c: c} rh.SetupRoutes() + return rh } @@ -116,6 +117,7 @@ type ImageTags struct { func (rh *RouteHandler) ListTags(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name, ok := vars["name"] + if !ok || name == "" { w.WriteHeader(http.StatusNotFound) return @@ -145,6 +147,7 @@ func (rh *RouteHandler) ListTags(w http.ResponseWriter, r *http.Request) { func (rh *RouteHandler) CheckManifest(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name, ok := vars["name"] + if !ok || name == "" { w.WriteHeader(http.StatusNotFound) return @@ -165,6 +168,7 @@ func (rh *RouteHandler) CheckManifest(w http.ResponseWriter, r *http.Request) { rh.c.Log.Error().Err(err).Msg("unexpected error") WriteJSON(w, http.StatusInternalServerError, NewError(MANIFEST_INVALID, map[string]string{"reference": reference})) } + return } @@ -193,6 +197,7 @@ type ImageManifest struct { func (rh *RouteHandler) GetManifest(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name, ok := vars["name"] + if !ok || name == "" { w.WriteHeader(http.StatusNotFound) return @@ -217,6 +222,7 @@ func (rh *RouteHandler) GetManifest(w http.ResponseWriter, r *http.Request) { rh.c.Log.Error().Err(err).Msg("unexpected error") w.WriteHeader(http.StatusInternalServerError) } + return } @@ -240,6 +246,7 @@ func (rh *RouteHandler) GetManifest(w http.ResponseWriter, r *http.Request) { func (rh *RouteHandler) UpdateManifest(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name, ok := vars["name"] + if !ok || name == "" { w.WriteHeader(http.StatusNotFound) return @@ -278,6 +285,7 @@ func (rh *RouteHandler) UpdateManifest(w http.ResponseWriter, r *http.Request) { rh.c.Log.Error().Err(err).Msg("unexpected error") w.WriteHeader(http.StatusInternalServerError) } + return } @@ -298,6 +306,7 @@ func (rh *RouteHandler) UpdateManifest(w http.ResponseWriter, r *http.Request) { func (rh *RouteHandler) DeleteManifest(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name, ok := vars["name"] + if !ok || name == "" { w.WriteHeader(http.StatusNotFound) return @@ -320,6 +329,7 @@ func (rh *RouteHandler) DeleteManifest(w http.ResponseWriter, r *http.Request) { rh.c.Log.Error().Err(err).Msg("unexpected error") w.WriteHeader(http.StatusInternalServerError) } + return } @@ -339,6 +349,7 @@ func (rh *RouteHandler) DeleteManifest(w http.ResponseWriter, r *http.Request) { func (rh *RouteHandler) CheckBlob(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name, ok := vars["name"] + if !ok || name == "" { w.WriteHeader(http.StatusNotFound) return @@ -365,6 +376,7 @@ func (rh *RouteHandler) CheckBlob(w http.ResponseWriter, r *http.Request) { rh.c.Log.Error().Err(err).Msg("unexpected error") w.WriteHeader(http.StatusInternalServerError) } + return } @@ -391,6 +403,7 @@ func (rh *RouteHandler) CheckBlob(w http.ResponseWriter, r *http.Request) { func (rh *RouteHandler) GetBlob(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name, ok := vars["name"] + if !ok || name == "" { w.WriteHeader(http.StatusNotFound) return @@ -417,6 +430,7 @@ func (rh *RouteHandler) GetBlob(w http.ResponseWriter, r *http.Request) { rh.c.Log.Error().Err(err).Msg("unexpected error") w.WriteHeader(http.StatusInternalServerError) } + return } @@ -438,6 +452,7 @@ func (rh *RouteHandler) GetBlob(w http.ResponseWriter, r *http.Request) { func (rh *RouteHandler) DeleteBlob(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name, ok := vars["name"] + if !ok || name == "" { w.WriteHeader(http.StatusNotFound) return @@ -462,6 +477,7 @@ func (rh *RouteHandler) DeleteBlob(w http.ResponseWriter, r *http.Request) { rh.c.Log.Error().Err(err).Msg("unexpected error") w.WriteHeader(http.StatusInternalServerError) } + return } @@ -483,6 +499,7 @@ func (rh *RouteHandler) DeleteBlob(w http.ResponseWriter, r *http.Request) { func (rh *RouteHandler) CreateBlobUpload(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name, ok := vars["name"] + if !ok || name == "" { w.WriteHeader(http.StatusNotFound) return @@ -497,6 +514,7 @@ func (rh *RouteHandler) CreateBlobUpload(w http.ResponseWriter, r *http.Request) rh.c.Log.Error().Err(err).Msg("unexpected error") w.WriteHeader(http.StatusInternalServerError) } + return } @@ -521,6 +539,7 @@ func (rh *RouteHandler) CreateBlobUpload(w http.ResponseWriter, r *http.Request) func (rh *RouteHandler) GetBlobUpload(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name, ok := vars["name"] + if !ok || name == "" { w.WriteHeader(http.StatusNotFound) return @@ -547,6 +566,7 @@ func (rh *RouteHandler) GetBlobUpload(w http.ResponseWriter, r *http.Request) { rh.c.Log.Error().Err(err).Msg("unexpected error") w.WriteHeader(http.StatusInternalServerError) } + return } @@ -575,10 +595,12 @@ func (rh *RouteHandler) PatchBlobUpload(w http.ResponseWriter, r *http.Request) rh.c.Log.Info().Interface("headers", r.Header).Msg("request headers") vars := mux.Vars(r) name, ok := vars["name"] + if !ok || name == "" { w.WriteHeader(http.StatusNotFound) return } + uuid, ok := vars["uuid"] if !ok || uuid == "" { w.WriteHeader(http.StatusNotFound) @@ -586,10 +608,13 @@ func (rh *RouteHandler) PatchBlobUpload(w http.ResponseWriter, r *http.Request) } var err error + var contentLength int64 + if contentLength, err = strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64); err != nil { rh.c.Log.Warn().Str("actual", r.Header.Get("Content-Length")).Msg("invalid content length") w.WriteHeader(http.StatusBadRequest) + return } @@ -597,6 +622,7 @@ func (rh *RouteHandler) PatchBlobUpload(w http.ResponseWriter, r *http.Request) if contentRange == "" { rh.c.Log.Warn().Str("actual", r.Header.Get("Content-Range")).Msg("invalid content range") w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) + return } @@ -609,6 +635,7 @@ func (rh *RouteHandler) PatchBlobUpload(w http.ResponseWriter, r *http.Request) if contentType := r.Header.Get("Content-Type"); contentType != "application/octet-stream" { rh.c.Log.Warn().Str("actual", contentType).Str("expected", "application/octet-stream").Msg("invalid media type") w.WriteHeader(http.StatusUnsupportedMediaType) + return } @@ -625,6 +652,7 @@ func (rh *RouteHandler) PatchBlobUpload(w http.ResponseWriter, r *http.Request) rh.c.Log.Error().Err(err).Msg("unexpected error") w.WriteHeader(http.StatusInternalServerError) } + return } @@ -652,6 +680,7 @@ func (rh *RouteHandler) PatchBlobUpload(w http.ResponseWriter, r *http.Request) func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name, ok := vars["name"] + if !ok || name == "" { w.WriteHeader(http.StatusNotFound) return @@ -668,14 +697,18 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request) w.WriteHeader(http.StatusBadRequest) return } + digest := digests[0] contentPresent := true contentLen, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64) + if err != nil || contentLen == 0 { contentPresent = false } + contentRangePresent := true + if r.Header.Get("Content-Range") == "" { contentRangePresent = false } @@ -698,10 +731,12 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request) contentRange := r.Header.Get("Content-Range") if contentRange == "" { // monolithic upload from = 0 + if contentLen == 0 { w.WriteHeader(http.StatusBadRequest) return } + to = contentLen } else if from, to, err = getContentRange(r); err != nil { // finish chunked upload w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) @@ -721,6 +756,7 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request) rh.c.Log.Error().Err(err).Msg("unexpected error") w.WriteHeader(http.StatusInternalServerError) } + return } } @@ -740,6 +776,7 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request) rh.c.Log.Error().Err(err).Msg("unexpected error") w.WriteHeader(http.StatusInternalServerError) } + return } @@ -763,6 +800,7 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request) func (rh *RouteHandler) DeleteBlobUpload(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name, ok := vars["name"] + if !ok || name == "" { w.WriteHeader(http.StatusNotFound) return @@ -784,6 +822,7 @@ func (rh *RouteHandler) DeleteBlobUpload(w http.ResponseWriter, r *http.Request) rh.c.Log.Error().Err(err).Msg("unexpected error") w.WriteHeader(http.StatusInternalServerError) } + return } @@ -820,25 +859,31 @@ func getContentRange(r *http.Request) (int64 /* from */, int64 /* to */, error) contentRange := r.Header.Get("Content-Range") tokens := strings.Split(contentRange, "-") from, err := strconv.ParseInt(tokens[0], 10, 64) + if err != nil { return -1, -1, errors.ErrBadUploadRange } + to, err := strconv.ParseInt(tokens[1], 10, 64) if err != nil { return -1, -1, errors.ErrBadUploadRange } + if from > to { return -1, -1, errors.ErrBadUploadRange } + return from, to, nil } func WriteJSON(w http.ResponseWriter, status int, data interface{}) { var json = jsoniter.ConfigCompatibleWithStandardLibrary body, err := json.Marshal(data) + if err != nil { w.WriteHeader(http.StatusInternalServerError) } + WriteData(w, status, DefaultMediaType, body) } @@ -853,6 +898,7 @@ func WriteDataFromReader(w http.ResponseWriter, status int, length int64, mediaT w.Header().Set("Content-Length", strconv.FormatInt(length, 10)) const maxSize = 10 * 1024 * 1024 + for { size, err := io.CopyN(w, reader, maxSize) if size == 0 { @@ -860,6 +906,7 @@ func WriteDataFromReader(w http.ResponseWriter, status int, length int64, mediaT w.WriteHeader(http.StatusInternalServerError) return } + break } } diff --git a/pkg/cli/root.go b/pkg/cli/root.go index 66b65f3c..d5e969dc 100644 --- a/pkg/cli/root.go +++ b/pkg/cli/root.go @@ -80,6 +80,7 @@ func NewRootCmd() *cobra.Command { gcCmd.Flags().StringVarP(&config.Storage.RootDirectory, "storage-root-dir", "r", "", "Use specified directory for filestore backing image data") + _ = gcCmd.MarkFlagRequired("storage-root-dir") gcCmd.Flags().BoolVarP(&gcDelUntagged, "delete-untagged", "m", false, "delete manifests that are not currently referenced via tag") @@ -106,14 +107,18 @@ func NewRootCmd() *cobra.Command { complianceCmd.Flags().StringVarP(&complianceConfig.Address, "address", "H", "", "Registry server address") + if err := complianceCmd.MarkFlagRequired("address"); err != nil { panic(err) } + complianceCmd.Flags().StringVarP(&complianceConfig.Port, "port", "P", "", "Registry server port") + if err := complianceCmd.MarkFlagRequired("port"); err != nil { panic(err) } + complianceCmd.Flags().StringVarP(&complianceConfig.Version, "version", "V", "all", "OCI dist-spec version to check") diff --git a/pkg/cli/root_test.go b/pkg/cli/root_test.go index 31d32ee4..9e10578d 100644 --- a/pkg/cli/root_test.go +++ b/pkg/cli/root_test.go @@ -11,6 +11,7 @@ import ( func TestUsage(t *testing.T) { oldArgs := os.Args + defer func() { os.Args = oldArgs }() Convey("Test usage", t, func(c C) { @@ -28,6 +29,7 @@ func TestUsage(t *testing.T) { func TestServe(t *testing.T) { oldArgs := os.Args + defer func() { os.Args = oldArgs }() Convey("Test serve help", t, func(c C) { @@ -64,6 +66,7 @@ func TestServe(t *testing.T) { func TestGC(t *testing.T) { oldArgs := os.Args + defer func() { os.Args = oldArgs }() Convey("Test gc", t, func(c C) { diff --git a/pkg/compliance/v1_0_0/check_test.go b/pkg/compliance/v1_0_0/check_test.go index 337b0ccf..be866fb6 100644 --- a/pkg/compliance/v1_0_0/check_test.go +++ b/pkg/compliance/v1_0_0/check_test.go @@ -29,11 +29,13 @@ func TestMain(m *testing.M) { config.HTTP.Port = Port c := api.NewController(config) dir, err := ioutil.TempDir("", "oci-repo-test") + if err != nil { panic(err) } //defer os.RemoveAll(dir) c.Config.Storage.RootDirectory = dir + go func() { // this blocks if err := c.Run(); err != nil { @@ -42,16 +44,20 @@ func TestMain(m *testing.M) { }() BaseURL := fmt.Sprintf("http://%s:%s", Address, Port) + for { // poll until ready resp, _ := resty.R().Get(BaseURL) if resp.StatusCode() == 404 { break } + time.Sleep(100 * time.Millisecond) } + status := m.Run() ctx := context.Background() _ = c.Server.Shutdown(ctx) + os.Exit(status) } diff --git a/pkg/log/log.go b/pkg/log/log.go index fdbb05b3..5188e3c0 100644 --- a/pkg/log/log.go +++ b/pkg/log/log.go @@ -21,11 +21,15 @@ func (l Logger) Println(v ...interface{}) { func NewLogger(level string, output string) Logger { zerolog.TimeFieldFormat = time.RFC3339Nano lvl, err := zerolog.ParseLevel(level) + if err != nil { panic(err) } + zerolog.SetGlobalLevel(lvl) + var log zerolog.Logger + if output == "" { log = zerolog.New(os.Stdout) } else { @@ -35,6 +39,7 @@ func NewLogger(level string, output string) Logger { } log = zerolog.New(file) } + return Logger{Logger: log.With().Timestamp().Logger()} } @@ -53,13 +58,16 @@ func (w *statusWriter) Write(b []byte) (int, error) { if w.status == 0 { w.status = 200 } + n, err := w.ResponseWriter.Write(b) w.length += n + return n, err } func SessionLogger(log Logger) mux.MiddlewareFunc { l := log.With().Str("module", "http").Logger() + return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Start timer diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index de61a18d..f9f57362 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -39,6 +39,7 @@ func NewImageStore(rootDir string, log zlog.Logger) *ImageStore { blobUploads: make(map[string]BlobUpload), log: log.With().Caller().Logger(), } + if _, err := os.Stat(rootDir); os.IsNotExist(err) { _ = os.MkdirAll(rootDir, 0700) } @@ -69,9 +70,11 @@ func (is *ImageStore) InitRepo(name string) error { if _, err := os.Stat(ilPath); err != nil { il := ispec.ImageLayout{Version: ispec.ImageLayoutVersion} buf, err := json.Marshal(il) + if err != nil { is.log.Panic().Err(err).Msg("unable to marshal JSON") } + if err := ioutil.WriteFile(ilPath, buf, 0644); err != nil { is.log.Panic().Err(err).Str("file", ilPath).Msg("unable to write file") } @@ -83,9 +86,11 @@ func (is *ImageStore) InitRepo(name string) error { index := ispec.Index{} index.SchemaVersion = 2 buf, err := json.Marshal(index) + if err != nil { is.log.Panic().Err(err).Msg("unable to marshal JSON") } + if err := ioutil.WriteFile(indexPath, buf, 0644); err != nil { is.log.Panic().Err(err).Str("file", indexPath).Msg("unable to write file") } @@ -124,6 +129,7 @@ func (is *ImageStore) ValidateRepo(name string) (bool, error) { if file.Name() == "blobs" && !file.IsDir() { return false, nil } + found[file.Name()] = true } @@ -192,7 +198,9 @@ func (is *ImageStore) GetImageTags(repo string) ([]string, error) { if !dirExists(dir) { return nil, errors.ErrRepoNotFound } + buf, err := ioutil.ReadFile(path.Join(dir, "index.json")) + if err != nil { is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") return nil, errors.ErrRepoNotFound @@ -221,7 +229,9 @@ func (is *ImageStore) GetImageManifest(repo string, reference string) ([]byte, s if !dirExists(dir) { return nil, "", "", errors.ErrRepoNotFound } + buf, err := ioutil.ReadFile(path.Join(dir, "index.json")) + if err != nil { is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") return nil, "", "", err @@ -234,13 +244,17 @@ func (is *ImageStore) GetImageManifest(repo string, reference string) ([]byte, s } found := false + var digest godigest.Digest + mediaType := "" + for _, m := range index.Manifests { if reference == m.Digest.String() { digest = m.Digest mediaType = m.MediaType found = true + break } @@ -249,6 +263,7 @@ func (is *ImageStore) GetImageManifest(repo string, reference string) ([]byte, s digest = m.Digest mediaType = m.MediaType found = true + break } } @@ -276,9 +291,7 @@ func (is *ImageStore) GetImageManifest(repo string, reference string) ([]byte, s return buf, digest.String(), mediaType, nil } -func (is *ImageStore) PutImageManifest(repo string, reference string, - mediaType string, body []byte) (string, error) { - +func (is *ImageStore) PutImageManifest(repo string, reference string, mediaType string, body []byte) (string, error) { if err := is.InitRepo(repo); err != nil { return "", err } @@ -299,6 +312,7 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, for _, l := range m.Layers { digest := l.Digest blobPath := is.BlobPath(repo, digest) + if _, err := os.Stat(blobPath); err != nil { return digest.String(), errors.ErrBlobNotFound } @@ -307,17 +321,20 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, mDigest := godigest.FromBytes(body) refIsDigest := false d, err := godigest.Parse(reference) + if err == nil { if d.String() != mDigest.String() { is.log.Error().Str("actual", mDigest.String()).Str("expected", d.String()). Msg("manifest digest is not valid") return "", errors.ErrBadManifest } + refIsDigest = true } dir := path.Join(is.rootDir, repo) buf, err := ioutil.ReadFile(path.Join(dir, "index.json")) + if err != nil { is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") return "", err @@ -342,6 +359,7 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, // nothing changed, so don't update desc = m updateIndex = false + break } @@ -351,12 +369,15 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, // nothing changed, so don't update desc = m updateIndex = false + break } // manifest contents have changed for the same tag desc = m desc.Digest = mDigest + index.Manifests = append(index.Manifests[:i], index.Manifests[i+1:]...) + break } } @@ -371,6 +392,7 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, dir = path.Join(dir, mDigest.Algorithm().String()) _ = os.MkdirAll(dir, 0755) file := path.Join(dir, mDigest.Encoded()) + if err := ioutil.WriteFile(file, body, 0644); err != nil { return "", err } @@ -380,9 +402,11 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, dir = path.Join(is.rootDir, repo) file = path.Join(dir, "index.json") buf, err = json.Marshal(index) + if err != nil { return "", err } + if err := ioutil.WriteFile(file, buf, 0644); err != nil { return "", err } @@ -395,7 +419,9 @@ func (is *ImageStore) DeleteImageManifest(repo string, reference string) error { if !dirExists(dir) { return errors.ErrRepoNotFound } + buf, err := ioutil.ReadFile(path.Join(dir, "index.json")) + if err != nil { is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") return err @@ -408,13 +434,18 @@ func (is *ImageStore) DeleteImageManifest(repo string, reference string) error { } found := false + var digest godigest.Digest + var i int + var m ispec.Descriptor + for i, m = range index.Manifests { if reference == m.Digest.String() { digest = m.Digest found = true + break } @@ -422,6 +453,7 @@ func (is *ImageStore) DeleteImageManifest(repo string, reference string) error { if ok && v == reference { digest = m.Digest found = true + break } } @@ -438,9 +470,11 @@ func (is *ImageStore) DeleteImageManifest(repo string, reference string) error { dir = path.Join(is.rootDir, repo) file := path.Join(dir, "index.json") buf, err = json.Marshal(index) + if err != nil { return err } + if err := ioutil.WriteFile(file, buf, 0644); err != nil { return err } @@ -458,6 +492,7 @@ func (is *ImageStore) BlobUploadPath(repo string, uuid string) string { dir := path.Join(is.rootDir, repo) blobUploadPath := path.Join(dir, BlobUploadDir) blobUploadPath = path.Join(blobUploadPath, uuid) + return blobUploadPath } @@ -474,6 +509,7 @@ func (is *ImageStore) NewBlobUpload(repo string) (string, error) { u := uuid.String() blobUploadPath := is.BlobUploadPath(repo, u) file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0600) + if err != nil { return "", errors.ErrRepoNotFound } @@ -485,19 +521,19 @@ func (is *ImageStore) NewBlobUpload(repo string) (string, error) { func (is *ImageStore) GetBlobUpload(repo string, uuid string) (int64, error) { blobUploadPath := is.BlobUploadPath(repo, uuid) fi, err := os.Stat(blobUploadPath) + if err != nil { if os.IsNotExist(err) { return -1, errors.ErrUploadNotFound } + return -1, err } return fi.Size(), nil } -func (is *ImageStore) PutBlobChunk(repo string, uuid string, - from int64, to int64, body io.Reader) (int64, error) { - +func (is *ImageStore) PutBlobChunk(repo string, uuid string, from int64, to int64, body io.Reader) (int64, error) { if err := is.InitRepo(repo); err != nil { return -1, err } @@ -508,6 +544,7 @@ func (is *ImageStore) PutBlobChunk(repo string, uuid string, if err != nil { return -1, errors.ErrUploadNotFound } + if from != fi.Size() { is.log.Error().Int64("expected", from).Int64("actual", fi.Size()). Msg("invalid range start for blob upload") @@ -529,23 +566,25 @@ func (is *ImageStore) PutBlobChunk(repo string, uuid string, } n, err := io.Copy(file, body) + return n, err } func (is *ImageStore) BlobUploadInfo(repo string, uuid string) (int64, error) { blobUploadPath := is.BlobUploadPath(repo, uuid) fi, err := os.Stat(blobUploadPath) + if err != nil { is.log.Error().Err(err).Str("blob", blobUploadPath).Msg("failed to stat blob") return -1, err } + size := fi.Size() + return size, nil } -func (is *ImageStore) FinishBlobUpload(repo string, uuid string, - body io.Reader, digest string) error { - +func (is *ImageStore) FinishBlobUpload(repo string, uuid string, body io.Reader, digest string) error { dstDigest, err := godigest.Parse(digest) if err != nil { is.log.Error().Err(err).Str("digest", digest).Msg("failed to parse digest") @@ -565,8 +604,10 @@ func (is *ImageStore) FinishBlobUpload(repo string, uuid string, is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") return errors.ErrUploadNotFound } + srcDigest, err := godigest.FromReader(f) f.Close() + if err != nil { is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") return errors.ErrBadBlobDigest @@ -593,6 +634,7 @@ func (is *ImageStore) FinishBlobUpload(repo string, uuid string, func (is *ImageStore) DeleteBlobUpload(repo string, uuid string) error { blobUploadPath := is.BlobUploadPath(repo, uuid) _ = os.Remove(blobUploadPath) + return nil } @@ -601,12 +643,12 @@ func (is *ImageStore) BlobPath(repo string, digest godigest.Digest) string { blobPath := path.Join(dir, "blobs") blobPath = path.Join(blobPath, digest.Algorithm().String()) blobPath = path.Join(blobPath, digest.Encoded()) + return blobPath } func (is *ImageStore) CheckBlob(repo string, digest string, mediaType string) (bool, int64, error) { - d, err := godigest.Parse(digest) if err != nil { is.log.Error().Err(err).Str("digest", digest).Msg("failed to parse digest") @@ -626,9 +668,7 @@ func (is *ImageStore) CheckBlob(repo string, digest string, // FIXME: we should probably parse the manifest and use (digest, mediaType) as a // blob selector instead of directly downloading the blob -func (is *ImageStore) GetBlob(repo string, digest string, - mediaType string) (io.Reader, int64, error) { - +func (is *ImageStore) GetBlob(repo string, digest string, mediaType string) (io.Reader, int64, error) { d, err := godigest.Parse(digest) if err != nil { is.log.Error().Err(err).Str("digest", digest).Msg("failed to parse digest") @@ -687,9 +727,11 @@ func dirExists(d string) bool { if err != nil && os.IsNotExist(err) { return false } + if !fi.IsDir() { return false } + return true } diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 0e0dd016..8283e6de 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -21,6 +21,7 @@ func TestAPIs(t *testing.T) { if err != nil { panic(err) } + defer os.RemoveAll(dir) il := storage.NewImageStore(dir, log.Logger{Logger: zerolog.New(os.Stdout)})