//go:build search // +build search package client import ( "bytes" "context" "encoding/json" "fmt" "io" "net/http" "net/url" "strconv" "strings" "sync" "time" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sigstore/cosign/v2/pkg/oci/remote" zerr "zotregistry.dev/zot/errors" "zotregistry.dev/zot/pkg/common" ) var ( httpClientsMap = make(map[string]*http.Client) //nolint: gochecknoglobals httpClientLock sync.Mutex //nolint: gochecknoglobals ) func makeGETRequest(ctx context.Context, url, username, password string, verifyTLS bool, debug bool, resultsPtr interface{}, configWriter io.Writer, ) (http.Header, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, err } req.SetBasicAuth(username, password) return doHTTPRequest(req, verifyTLS, debug, resultsPtr, configWriter) } func makeHEADRequest(ctx context.Context, url, username, password string, verifyTLS bool, debug bool, ) (http.Header, error) { req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil) if err != nil { return nil, err } req.SetBasicAuth(username, password) return doHTTPRequest(req, verifyTLS, debug, nil, io.Discard) } func makeGraphQLRequest(ctx context.Context, url, query, username, password string, verifyTLS bool, debug bool, resultsPtr interface{}, configWriter io.Writer, ) error { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, bytes.NewBufferString(query)) if err != nil { return err } q := req.URL.Query() q.Add("query", query) req.URL.RawQuery = q.Encode() req.SetBasicAuth(username, password) req.Header.Add("Content-Type", "application/json") _, err = doHTTPRequest(req, verifyTLS, debug, resultsPtr, configWriter) if err != nil { return err } return nil } func doHTTPRequest(req *http.Request, verifyTLS bool, debug bool, resultsPtr interface{}, configWriter io.Writer, ) (http.Header, error) { var httpClient *http.Client var err error host := req.Host httpClientLock.Lock() if httpClientsMap[host] == nil { httpClient, err = common.CreateHTTPClient(verifyTLS, host, "") if err != nil { return nil, err } httpClientsMap[host] = httpClient } else { httpClient = httpClientsMap[host] } httpClientLock.Unlock() if debug { fmt.Fprintln(configWriter, "[debug] ", req.Method, " ", req.URL, "[request header] ", req.Header) } resp, err := httpClient.Do(req) if err != nil { return nil, err } if debug { fmt.Fprintln(configWriter, "[debug] ", req.Method, req.URL, "[status] ", resp.StatusCode, " ", "[response header] ", resp.Header) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { var err error switch resp.StatusCode { case http.StatusNotFound: err = zerr.ErrURLNotFound case http.StatusUnauthorized: err = zerr.ErrUnauthorizedAccess default: err = zerr.ErrBadHTTPStatusCode } bodyBytes, _ := io.ReadAll(resp.Body) return nil, fmt.Errorf("%w: Expected: %d, Got: %d, Body: '%s'", err, http.StatusOK, resp.StatusCode, string(bodyBytes)) } if resultsPtr == nil { return resp.Header, nil } if err := json.NewDecoder(resp.Body).Decode(resultsPtr); err != nil { return nil, err } return resp.Header, nil } func validateURL(str string) error { parsedURL, err := url.Parse(str) if err != nil { if strings.Contains(err.Error(), "first path segment in URL cannot contain colon") { return fmt.Errorf("%w: scheme not provided (ex: https://)", zerr.ErrInvalidURL) } return err } if parsedURL.Scheme == "" || parsedURL.Host == "" { return fmt.Errorf("%w: scheme not provided (ex: https://)", zerr.ErrInvalidURL) } return nil } type requestsPool struct { jobs chan *httpJob done chan struct{} wtgrp *sync.WaitGroup outputCh chan stringResult } type httpJob struct { url string username string password string imageName string tagName string config SearchConfig } const rateLimiterBuffer = 5000 func newSmoothRateLimiter(wtgrp *sync.WaitGroup, opch chan stringResult) *requestsPool { ch := make(chan *httpJob, rateLimiterBuffer) return &requestsPool{ jobs: ch, done: make(chan struct{}), wtgrp: wtgrp, outputCh: opch, } } // block every "rateLimit" time duration. const rateLimit = 100 * time.Millisecond func (p *requestsPool) startRateLimiter(ctx context.Context) { p.wtgrp.Done() throttle := time.NewTicker(rateLimit).C for { select { case job := <-p.jobs: go p.doJob(ctx, job) case <-p.done: return } <-throttle } } func (p *requestsPool) doJob(ctx context.Context, job *httpJob) { defer p.wtgrp.Done() // Check manifest media type header, err := makeHEADRequest(ctx, job.url, job.username, job.password, job.config.VerifyTLS, job.config.Debug) if err != nil { if common.IsContextDone(ctx) { return } p.outputCh <- stringResult{"", err} } verbose := job.config.Verbose switch header.Get("Content-Type") { case ispec.MediaTypeImageManifest: image, err := fetchImageManifestStruct(ctx, job) if err != nil { if common.IsContextDone(ctx) { return } p.outputCh <- stringResult{"", err} return } platformStr := getPlatformStr(image.Manifests[0].Platform) str, err := image.string(job.config.OutputFormat, len(job.imageName), len(job.tagName), len(platformStr), verbose) if err != nil { if common.IsContextDone(ctx) { return } p.outputCh <- stringResult{"", err} return } if common.IsContextDone(ctx) { return } p.outputCh <- stringResult{str, nil} case ispec.MediaTypeImageIndex: image, err := fetchImageIndexStruct(ctx, job) if err != nil { if common.IsContextDone(ctx) { return } p.outputCh <- stringResult{"", err} return } platformStr := getPlatformStr(image.Manifests[0].Platform) str, err := image.string(job.config.OutputFormat, len(job.imageName), len(job.tagName), len(platformStr), verbose) if err != nil { if common.IsContextDone(ctx) { return } p.outputCh <- stringResult{"", err} return } if common.IsContextDone(ctx) { return } p.outputCh <- stringResult{str, nil} default: return } } func fetchImageIndexStruct(ctx context.Context, job *httpJob) (*imageStruct, error) { var indexContent ispec.Index header, err := makeGETRequest(ctx, job.url, job.username, job.password, job.config.VerifyTLS, job.config.Debug, &indexContent, job.config.ResultWriter) if err != nil { if common.IsContextDone(ctx) { return nil, context.Canceled } return nil, err } indexDigest := header.Get("docker-content-digest") indexSize, err := strconv.ParseInt(header.Get("Content-Length"), 10, 64) if err != nil { return nil, err } imageSize := indexSize manifestList := make([]common.ManifestSummary, 0, len(indexContent.Manifests)) for _, manifestDescriptor := range indexContent.Manifests { manifest, err := fetchManifestStruct(ctx, job.imageName, manifestDescriptor.Digest.String(), job.config, job.username, job.password) if err != nil { return nil, err } imageSize += int64(atoiWithDefault(manifest.Size, 0)) if manifestDescriptor.Platform != nil { manifest.Platform = common.Platform{ Os: manifestDescriptor.Platform.OS, Arch: manifestDescriptor.Platform.Architecture, Variant: manifestDescriptor.Platform.Variant, } } manifestList = append(manifestList, manifest) } isIndexSigned := isCosignSigned(ctx, job.imageName, indexDigest, job.config, job.username, job.password) || isNotationSigned(ctx, job.imageName, indexDigest, job.config, job.username, job.password) return &imageStruct{ RepoName: job.imageName, Tag: job.tagName, Digest: indexDigest, MediaType: ispec.MediaTypeImageIndex, Manifests: manifestList, Size: strconv.FormatInt(imageSize, 10), IsSigned: isIndexSigned, }, nil } func atoiWithDefault(size string, defaultVal int) int { val, err := strconv.Atoi(size) if err != nil { return defaultVal } return val } func fetchImageManifestStruct(ctx context.Context, job *httpJob) (*imageStruct, error) { manifest, err := fetchManifestStruct(ctx, job.imageName, job.tagName, job.config, job.username, job.password) if err != nil { return nil, err } return &imageStruct{ RepoName: job.imageName, Tag: job.tagName, Digest: manifest.Digest, MediaType: ispec.MediaTypeImageManifest, Manifests: []common.ManifestSummary{ manifest, }, Size: manifest.Size, IsSigned: manifest.IsSigned, }, nil } func fetchManifestStruct(ctx context.Context, repo, manifestReference string, searchConf SearchConfig, username, password string, ) (common.ManifestSummary, error) { manifestResp := ispec.Manifest{} URL := fmt.Sprintf("%s/v2/%s/manifests/%s", searchConf.ServURL, repo, manifestReference) header, err := makeGETRequest(ctx, URL, username, password, searchConf.VerifyTLS, searchConf.Debug, &manifestResp, searchConf.ResultWriter) if err != nil { if common.IsContextDone(ctx) { return common.ManifestSummary{}, context.Canceled } return common.ManifestSummary{}, err } manifestDigest := header.Get("docker-content-digest") configDigest := manifestResp.Config.Digest.String() configContent, err := fetchConfig(ctx, repo, configDigest, searchConf, username, password) if err != nil { if common.IsContextDone(ctx) { return common.ManifestSummary{}, context.Canceled } return common.ManifestSummary{}, err } opSys := "" arch := "" variant := "" if manifestResp.Config.Platform != nil { opSys = manifestResp.Config.Platform.OS arch = manifestResp.Config.Platform.Architecture variant = manifestResp.Config.Platform.Variant } if opSys == "" { opSys = configContent.OS } if arch == "" { arch = configContent.Architecture } if variant == "" { variant = configContent.Variant } manifestSize, err := strconv.ParseInt(header.Get("Content-Length"), 10, 64) if err != nil { return common.ManifestSummary{}, err } var imageSize int64 imageSize += manifestResp.Config.Size imageSize += manifestSize layers := []common.LayerSummary{} for _, entry := range manifestResp.Layers { imageSize += entry.Size layers = append( layers, common.LayerSummary{ Size: fmt.Sprintf("%v", entry.Size), Digest: entry.Digest.String(), }, ) } isSigned := isCosignSigned(ctx, repo, manifestDigest, searchConf, username, password) || isNotationSigned(ctx, repo, manifestDigest, searchConf, username, password) return common.ManifestSummary{ ConfigDigest: configDigest, Digest: manifestDigest, Layers: layers, Platform: common.Platform{Os: opSys, Arch: arch, Variant: variant}, Size: strconv.FormatInt(imageSize, 10), IsSigned: isSigned, }, nil } func fetchConfig(ctx context.Context, repo, configDigest string, searchConf SearchConfig, username, password string, ) (ispec.Image, error) { configContent := ispec.Image{} URL := fmt.Sprintf("%s/v2/%s/blobs/%s", searchConf.ServURL, repo, configDigest) _, err := makeGETRequest(ctx, URL, username, password, searchConf.VerifyTLS, searchConf.Debug, &configContent, searchConf.ResultWriter) if err != nil { if common.IsContextDone(ctx) { return ispec.Image{}, context.Canceled } return ispec.Image{}, err } return configContent, nil } func isNotationSigned(ctx context.Context, repo, digestStr string, searchConf SearchConfig, username, password string, ) bool { var referrers ispec.Index URL := fmt.Sprintf("%s/v2/%s/referrers/%s?artifactType=%s", searchConf.ServURL, repo, digestStr, common.ArtifactTypeNotation) _, err := makeGETRequest(ctx, URL, username, password, searchConf.VerifyTLS, searchConf.Debug, &referrers, searchConf.ResultWriter) if err != nil { return false } if len(referrers.Manifests) > 0 { return true } return false } func isCosignSigned(ctx context.Context, repo, digestStr string, searchConf SearchConfig, username, password string, ) bool { var result interface{} cosignTag := strings.Replace(digestStr, ":", "-", 1) + "." + remote.SignatureTagSuffix URL := fmt.Sprintf("%s/v2/%s/manifests/%s", searchConf.ServURL, repo, cosignTag) _, err := makeGETRequest(ctx, URL, username, password, searchConf.VerifyTLS, searchConf.Debug, &result, searchConf.ResultWriter) if err == nil { return true } var referrers ispec.Index artifactType := url.QueryEscape(common.ArtifactTypeCosign) URL = fmt.Sprintf("%s/v2/%s/referrers/%s?artifactType=%s", searchConf.ServURL, repo, digestStr, artifactType) _, err = makeGETRequest(ctx, URL, username, password, searchConf.VerifyTLS, searchConf.Debug, &referrers, searchConf.ResultWriter) if err != nil { return false } if len(referrers.Manifests) == 0 { return false } return true } func (p *requestsPool) submitJob(job *httpJob) { p.jobs <- job }