2020-10-14 16:47:20 -05:00
|
|
|
// +build extended
|
|
|
|
|
2020-06-16 20:52:40 -05:00
|
|
|
package cli
|
|
|
|
|
|
|
|
import (
|
2020-07-06 17:44:32 -05:00
|
|
|
"bytes"
|
2020-06-16 20:52:40 -05:00
|
|
|
"context"
|
2020-07-17 14:42:22 -05:00
|
|
|
"crypto/tls"
|
2020-06-16 20:52:40 -05:00
|
|
|
"encoding/json"
|
|
|
|
"errors"
|
|
|
|
"io/ioutil"
|
|
|
|
"net/http"
|
|
|
|
"net/url"
|
|
|
|
"strings"
|
|
|
|
"sync"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
zotErrors "github.com/anuvu/zot/errors"
|
|
|
|
)
|
|
|
|
|
2020-07-17 14:42:22 -05:00
|
|
|
var httpClient *http.Client //nolint: gochecknoglobals
|
2020-06-16 20:52:40 -05:00
|
|
|
|
2020-07-06 17:44:32 -05:00
|
|
|
const httpTimeout = 5 * time.Minute
|
2020-06-16 20:52:40 -05:00
|
|
|
|
2020-07-17 14:42:22 -05:00
|
|
|
func createHTTPClient(verifyTLS bool) *http.Client {
|
|
|
|
var tr = http.DefaultTransport.(*http.Transport).Clone()
|
|
|
|
if !verifyTLS {
|
|
|
|
tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint: gosec
|
|
|
|
}
|
|
|
|
|
2020-06-16 20:52:40 -05:00
|
|
|
return &http.Client{
|
2020-07-17 14:42:22 -05:00
|
|
|
Timeout: httpTimeout,
|
|
|
|
Transport: tr,
|
2020-06-16 20:52:40 -05:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-07-17 14:42:22 -05:00
|
|
|
func makeGETRequest(url, username, password string, verifyTLS bool, resultsPtr interface{}) (http.Header, error) {
|
2020-06-16 20:52:40 -05:00
|
|
|
req, err := http.NewRequest("GET", url, nil)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
req.SetBasicAuth(username, password)
|
|
|
|
|
2020-07-06 17:44:32 -05:00
|
|
|
return doHTTPRequest(req, verifyTLS, resultsPtr)
|
|
|
|
}
|
|
|
|
|
|
|
|
func makeGraphQLRequest(url, query, username,
|
|
|
|
password string, verifyTLS bool, resultsPtr interface{}) error {
|
|
|
|
req, err := http.NewRequest("GET", url, bytes.NewBufferString(query))
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
q := req.URL.Query()
|
|
|
|
q.Add("query", query)
|
|
|
|
|
|
|
|
req.URL.RawQuery = q.Encode()
|
|
|
|
|
|
|
|
req.SetBasicAuth(username, password)
|
|
|
|
req.Header.Add("Content-Type", "application/json")
|
|
|
|
|
|
|
|
_, err = doHTTPRequest(req, verifyTLS, resultsPtr)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func doHTTPRequest(req *http.Request, verifyTLS bool, resultsPtr interface{}) (http.Header, error) {
|
2020-07-17 14:42:22 -05:00
|
|
|
if httpClient == nil {
|
|
|
|
httpClient = createHTTPClient(verifyTLS)
|
|
|
|
}
|
|
|
|
|
2020-06-16 20:52:40 -05:00
|
|
|
resp, err := httpClient.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
defer resp.Body.Close()
|
|
|
|
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
|
|
if resp.StatusCode == http.StatusUnauthorized {
|
|
|
|
return nil, zotErrors.ErrUnauthorizedAccess
|
|
|
|
}
|
|
|
|
|
|
|
|
bodyBytes, _ := ioutil.ReadAll(resp.Body)
|
|
|
|
|
|
|
|
return nil, errors.New(string(bodyBytes)) //nolint: goerr113
|
|
|
|
}
|
|
|
|
|
|
|
|
if err := json.NewDecoder(resp.Body).Decode(resultsPtr); err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return resp.Header, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func isURL(str string) bool {
|
|
|
|
u, err := url.Parse(str)
|
|
|
|
return err == nil && u.Scheme != "" && u.Host != ""
|
|
|
|
} // from https://stackoverflow.com/a/55551215
|
|
|
|
|
|
|
|
type requestsPool struct {
|
|
|
|
jobs chan *manifestJob
|
|
|
|
done chan struct{}
|
|
|
|
waitGroup *sync.WaitGroup
|
2020-07-06 17:44:32 -05:00
|
|
|
outputCh chan stringResult
|
2020-06-16 20:52:40 -05:00
|
|
|
context context.Context
|
|
|
|
}
|
|
|
|
|
|
|
|
type manifestJob struct {
|
|
|
|
url string
|
|
|
|
username string
|
|
|
|
password string
|
|
|
|
imageName string
|
|
|
|
tagName string
|
2020-07-17 14:42:22 -05:00
|
|
|
config searchConfig
|
2020-06-16 20:52:40 -05:00
|
|
|
manifestResp manifestResponse
|
|
|
|
}
|
|
|
|
|
|
|
|
const rateLimiterBuffer = 5000
|
|
|
|
|
2020-07-06 17:44:32 -05:00
|
|
|
func newSmoothRateLimiter(ctx context.Context, wg *sync.WaitGroup, op chan stringResult) *requestsPool {
|
2020-06-16 20:52:40 -05:00
|
|
|
ch := make(chan *manifestJob, rateLimiterBuffer)
|
|
|
|
|
|
|
|
return &requestsPool{
|
|
|
|
jobs: ch,
|
|
|
|
done: make(chan struct{}),
|
|
|
|
waitGroup: wg,
|
|
|
|
outputCh: op,
|
|
|
|
context: ctx,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// block every "rateLimit" time duration.
|
|
|
|
const rateLimit = 100 * time.Millisecond
|
|
|
|
|
|
|
|
func (p *requestsPool) startRateLimiter() {
|
|
|
|
p.waitGroup.Done()
|
|
|
|
|
|
|
|
throttle := time.NewTicker(rateLimit).C
|
|
|
|
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case job := <-p.jobs:
|
|
|
|
go p.doJob(job)
|
|
|
|
case <-p.done:
|
|
|
|
return
|
|
|
|
}
|
|
|
|
<-throttle
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func (p *requestsPool) doJob(job *manifestJob) {
|
|
|
|
defer p.waitGroup.Done()
|
|
|
|
|
2020-07-17 14:42:22 -05:00
|
|
|
header, err := makeGETRequest(job.url, job.username, job.password, *job.config.verifyTLS, &job.manifestResp)
|
2020-06-16 20:52:40 -05:00
|
|
|
if err != nil {
|
|
|
|
if isContextDone(p.context) {
|
|
|
|
return
|
|
|
|
}
|
2020-07-06 17:44:32 -05:00
|
|
|
p.outputCh <- stringResult{"", err}
|
2020-06-16 20:52:40 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
digest := header.Get("docker-content-digest")
|
|
|
|
digest = strings.TrimPrefix(digest, "sha256:")
|
|
|
|
|
|
|
|
var size uint64
|
|
|
|
|
|
|
|
for _, layer := range job.manifestResp.Layers {
|
|
|
|
size += layer.Size
|
|
|
|
}
|
|
|
|
|
|
|
|
image := &imageStruct{}
|
|
|
|
image.Name = job.imageName
|
|
|
|
image.Tags = []tags{
|
|
|
|
{
|
|
|
|
Name: job.tagName,
|
|
|
|
Digest: digest,
|
|
|
|
Size: size,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
2020-07-17 14:42:22 -05:00
|
|
|
str, err := image.string(*job.config.outputFormat)
|
2020-06-16 20:52:40 -05:00
|
|
|
if err != nil {
|
|
|
|
if isContextDone(p.context) {
|
|
|
|
return
|
|
|
|
}
|
2020-07-06 17:44:32 -05:00
|
|
|
p.outputCh <- stringResult{"", err}
|
2020-06-16 20:52:40 -05:00
|
|
|
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
if isContextDone(p.context) {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2020-07-06 17:44:32 -05:00
|
|
|
p.outputCh <- stringResult{str, nil}
|
2020-06-16 20:52:40 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
func (p *requestsPool) submitJob(job *manifestJob) {
|
|
|
|
p.jobs <- job
|
|
|
|
}
|