Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2025-01-20 22:52:51 -05:00
Code Issues Activity

Fix data races in tests closes #599, closes #598

Browse source 
Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
This commit is contained in:
Petu Eusebiu 2022-06-30 18:43:31 +03:00 • committed by Ramkumar Chinchani
parent ada21ed842
commit 7954add73a
2 changed files with 127 additions and 260 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

359
pkg/cli/extensions_test.go
View file

@ -277,15 +277,9 @@ func TestServeSyncExtension(t *testing.T) {
defer func() { os.Args = oldArgs }()
Convey("sync implicitly enabled", t, func(c C) {
port := GetFreePort()
baseURL := GetBaseURL(port)
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name()) // clean up
content := fmt.Sprintf(`{
content := `{
"storage": {
"rootDirectory": "/tmp/zot"
"rootDirectory": "%s"
},
"http": {
"address": "127.0.0.1",
@ -316,39 +310,18 @@ func TestServeSyncExtension(t *testing.T) {
}]
}
}
}`, port, logFile.Name())
}`
cfgfile, err := ioutil.TempFile("", "zot-test*.json")
data, err := runCLIWithConfig(t.TempDir(), content)
So(err, ShouldBeNil)
defer os.Remove(cfgfile.Name()) // clean up
_, err = cfgfile.Write([]byte(content))
So(err, ShouldBeNil)
err = cfgfile.Close()
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "serve", cfgfile.Name()}
go func() {
err = cli.NewServerRootCmd().Execute()
So(err, ShouldBeNil)
}()
WaitTillServerReady(baseURL)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
So(data, ShouldContainSubstring,
"\"Extensions\":{\"Search\":null,\"Sync\":{\"Enable\":true")
})
Convey("sync explicitly enabled", t, func(c C) {
port := GetFreePort()
baseURL := GetBaseURL(port)
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name()) // clean up
content := fmt.Sprintf(`{
content := `{
"storage": {
"rootDirectory": "/tmp/zot"
"rootDirectory": "%s"
},
"http": {
"address": "127.0.0.1",
@ -380,39 +353,18 @@ func TestServeSyncExtension(t *testing.T) {
}]
}
}
}`, port, logFile.Name())
}`
cfgfile, err := ioutil.TempFile("", "zot-test*.json")
data, err := runCLIWithConfig(t.TempDir(), content)
So(err, ShouldBeNil)
defer os.Remove(cfgfile.Name()) // clean up
_, err = cfgfile.Write([]byte(content))
So(err, ShouldBeNil)
err = cfgfile.Close()
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "serve", cfgfile.Name()}
go func() {
err = cli.NewServerRootCmd().Execute()
So(err, ShouldBeNil)
}()
WaitTillServerReady(baseURL)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
So(data, ShouldContainSubstring,
"\"Extensions\":{\"Search\":null,\"Sync\":{\"Enable\":true")
})
Convey("sync explicitly disabled", t, func(c C) {
port := GetFreePort()
baseURL := GetBaseURL(port)
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name()) // clean up
content := fmt.Sprintf(`{
content := `{
"storage": {
"rootDirectory": "/tmp/zot"
"rootDirectory": "%s"
},
"http": {
"address": "127.0.0.1",
@ -434,26 +386,11 @@ func TestServeSyncExtension(t *testing.T) {
}]
}
}
}`, port, logFile.Name())
}`
cfgfile, err := ioutil.TempFile("", "zot-test*.json")
data, err := runCLIWithConfig(t.TempDir(), content)
So(err, ShouldBeNil)
defer os.Remove(cfgfile.Name()) // clean up
_, err = cfgfile.Write([]byte(content))
So(err, ShouldBeNil)
err = cfgfile.Close()
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "serve", cfgfile.Name()}
go func() {
err = cli.NewServerRootCmd().Execute()
So(err, ShouldBeNil)
}()
WaitTillServerReady(baseURL)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
So(data, ShouldContainSubstring,
"\"Extensions\":{\"Search\":null,\"Sync\":{\"Enable\":false")
})
}
@ -464,15 +401,9 @@ func TestServeScrubExtension(t *testing.T) {
defer func() { os.Args = oldArgs }()
Convey("scrub enabled by scrub interval param set", t, func(c C) {
port := GetFreePort()
baseURL := GetBaseURL(port)
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name()) // clean up
content := fmt.Sprintf(`{
content := `{
"storage": {
"rootDirectory": "/tmp/zot"
"rootDirectory": "%s"
},
"http": {
"address": "127.0.0.1",
@ -487,44 +418,23 @@ func TestServeScrubExtension(t *testing.T) {
"interval": "1h"
}
}
}`, port, logFile.Name())
}`
cfgfile, err := ioutil.TempFile("", "zot-test*.json")
So(err, ShouldBeNil)
defer os.Remove(cfgfile.Name()) // clean up
_, err = cfgfile.Write([]byte(content))
So(err, ShouldBeNil)
err = cfgfile.Close()
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "serve", cfgfile.Name()}
go func() {
err = cli.NewServerRootCmd().Execute()
So(err, ShouldBeNil)
}()
WaitTillServerReady(baseURL)
data, err := os.ReadFile(logFile.Name())
data, err := runCLIWithConfig(t.TempDir(), content)
So(err, ShouldBeNil)
// Even if in config we specified scrub interval=1h, the minimum interval is 2h
So(string(data), ShouldContainSubstring,
So(data, ShouldContainSubstring,
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":{\"Interval\":3600000000000}") //nolint:lll // gofumpt conflicts with lll
So(string(data), ShouldContainSubstring,
So(data, ShouldContainSubstring,
"Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.")
So(string(data), ShouldContainSubstring, "Starting periodic background tasks for")
So(string(data), ShouldContainSubstring, "Finishing periodic background tasks for")
So(data, ShouldContainSubstring, "Starting periodic background tasks for")
So(data, ShouldContainSubstring, "Finishing periodic background tasks for")
})
Convey("scrub not enabled - scrub interval param not set", t, func(c C) {
port := GetFreePort()
baseURL := GetBaseURL(port)
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name()) // clean up
content := fmt.Sprintf(`{
content := `{
"storage": {
"rootDirectory": "/tmp/zot"
"rootDirectory": "%s"
},
"http": {
"address": "127.0.0.1",
@ -538,46 +448,25 @@ func TestServeScrubExtension(t *testing.T) {
"scrub": {
}
}
}`, port, logFile.Name())
}`
cfgfile, err := ioutil.TempFile("", "zot-test*.json")
data, err := runCLIWithConfig(t.TempDir(), content)
So(err, ShouldBeNil)
defer os.Remove(cfgfile.Name()) // clean up
_, err = cfgfile.Write([]byte(content))
So(err, ShouldBeNil)
err = cfgfile.Close()
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "serve", cfgfile.Name()}
go func() {
err = cli.NewServerRootCmd().Execute()
So(err, ShouldBeNil)
}()
WaitTillServerReady(baseURL)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
So(data, ShouldContainSubstring,
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":null}")
So(string(data), ShouldContainSubstring, "Scrub config not provided, skipping scrub")
So(string(data), ShouldNotContainSubstring,
So(data, ShouldContainSubstring, "Scrub config not provided, skipping scrub")
So(data, ShouldNotContainSubstring,
"Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.")
})
}
func TestServeSearchExtension(t *testing.T) {
func TestServeSearchEnabled(t *testing.T) {
oldArgs := os.Args
defer func() { os.Args = oldArgs }()
Convey("search implicitly enabled", t, func(c C) {
port := GetFreePort()
baseURL := GetBaseURL(port)
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name()) // clean up
content := fmt.Sprintf(`{
content := `{
"storage": {
"rootDirectory": "%s"
},
@ -593,38 +482,23 @@ func TestServeSearchExtension(t *testing.T) {
"search": {
}
}
}`, t.TempDir(), port, logFile.Name())
}`
cfgfile, err := ioutil.TempFile("", "zot-test*.json")
data, err := runCLIWithConfig(t.TempDir(), content)
So(err, ShouldBeNil)
defer os.Remove(cfgfile.Name()) // clean up
_, err = cfgfile.Write([]byte(content))
So(err, ShouldBeNil)
err = cfgfile.Close()
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "serve", cfgfile.Name()}
go func() {
err = cli.NewServerRootCmd().Execute()
So(err, ShouldBeNil)
}()
WaitTillServerReady(baseURL)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
So(data, ShouldContainSubstring,
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":86400000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") //nolint:lll // gofumpt conflicts with lll
So(string(data), ShouldContainSubstring, "updating the CVE database")
So(data, ShouldContainSubstring, "updating the CVE database")
})
}
func TestServeSearchEnabledCVE(t *testing.T) {
oldArgs := os.Args
defer func() { os.Args = oldArgs }()
Convey("search implicitly enabled with CVE param set", t, func(c C) {
port := GetFreePort()
baseURL := GetBaseURL(port)
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name()) // clean up
content := fmt.Sprintf(`{
content := `{
"storage": {
"rootDirectory": "%s"
},
@ -643,41 +517,26 @@ func TestServeSearchExtension(t *testing.T) {
}
}
}
}`, t.TempDir(), port, logFile.Name())
}`
cfgfile, err := ioutil.TempFile("", "zot-test*.json")
So(err, ShouldBeNil)
defer os.Remove(cfgfile.Name()) // clean up
_, err = cfgfile.Write([]byte(content))
So(err, ShouldBeNil)
err = cfgfile.Close()
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "serve", cfgfile.Name()}
go func() {
err = cli.NewServerRootCmd().Execute()
So(err, ShouldBeNil)
}()
WaitTillServerReady(baseURL)
data, err := os.ReadFile(logFile.Name())
data, err := runCLIWithConfig(t.TempDir(), content)
So(err, ShouldBeNil)
// Even if in config we specified updateInterval=1h, the minimum interval is 2h
So(string(data), ShouldContainSubstring,
So(data, ShouldContainSubstring,
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":3600000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") //nolint:lll // gofumpt conflicts with lll
So(string(data), ShouldContainSubstring, "updating the CVE database")
So(string(data), ShouldContainSubstring,
So(data, ShouldContainSubstring, "updating the CVE database")
So(data, ShouldContainSubstring,
"CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing.")
})
}
func TestServeSearchEnabledNoCVE(t *testing.T) {
oldArgs := os.Args
defer func() { os.Args = oldArgs }()
Convey("search explicitly enabled, but CVE parameter not set", t, func(c C) {
port := GetFreePort()
baseURL := GetBaseURL(port)
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name()) // clean up
content := fmt.Sprintf(`{
content := `{
"storage": {
"rootDirectory": "%s"
},
@ -694,38 +553,23 @@ func TestServeSearchExtension(t *testing.T) {
"enable": true
}
}
}`, t.TempDir(), port, logFile.Name())
}`
cfgfile, err := ioutil.TempFile("", "zot-test*.json")
data, err := runCLIWithConfig(t.TempDir(), content)
So(err, ShouldBeNil)
defer os.Remove(cfgfile.Name()) // clean up
_, err = cfgfile.Write([]byte(content))
So(err, ShouldBeNil)
err = cfgfile.Close()
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "serve", cfgfile.Name()}
go func() {
err = cli.NewServerRootCmd().Execute()
So(err, ShouldBeNil)
}()
WaitTillServerReady(baseURL)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
So(data, ShouldContainSubstring,
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":86400000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") //nolint:lll // gofumpt conflicts with lll
So(string(data), ShouldContainSubstring, "updating the CVE database")
So(data, ShouldContainSubstring, "updating the CVE database")
})
}
func TestServeSearchDisabled(t *testing.T) {
oldArgs := os.Args
defer func() { os.Args = oldArgs }()
Convey("search explicitly disabled", t, func(c C) {
port := GetFreePort()
baseURL := GetBaseURL(port)
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name()) // clean up
content := fmt.Sprintf(`{
content := `{
"storage": {
"rootDirectory": "%s"
},
@ -745,29 +589,64 @@ func TestServeSearchExtension(t *testing.T) {
}
}
}
}`, t.TempDir(), port, logFile.Name())
}`
cfgfile, err := ioutil.TempFile("", "zot-test*.json")
So(err, ShouldBeNil)
defer os.Remove(cfgfile.Name()) // clean up
_, err = cfgfile.Write([]byte(content))
So(err, ShouldBeNil)
err = cfgfile.Close()
data, err := runCLIWithConfig(t.TempDir(), content)
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "serve", cfgfile.Name()}
go func() {
err = cli.NewServerRootCmd().Execute()
So(err, ShouldBeNil)
}()
WaitTillServerReady(baseURL)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
So(data, ShouldContainSubstring,
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":10800000000000},\"Enable\":false},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") //nolint:lll // gofumpt conflicts with lll
So(string(data), ShouldContainSubstring, "CVE config not provided, skipping CVE update")
So(string(data), ShouldNotContainSubstring,
So(data, ShouldContainSubstring, "CVE config not provided, skipping CVE update")
So(data, ShouldNotContainSubstring,
"CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing.")
})
}
// run cli and return output.
func runCLIWithConfig(tempDir string, config string) (string, error) {
port := GetFreePort()
baseURL := GetBaseURL(port)
logFile, err := ioutil.TempFile("", "zot-log*.txt")
if err != nil {
return "", err
}
defer os.Remove(logFile.Name()) // clean up
cfgfile, err := ioutil.TempFile("", "zot-test*.json")
if err != nil {
return "", err
}
config = fmt.Sprintf(config, tempDir, port, logFile.Name())
defer os.Remove(cfgfile.Name()) // clean up
_, err = cfgfile.Write([]byte(config))
if err != nil {
return "", err
}
err = cfgfile.Close()
if err != nil {
return "", err
}
os.Args = []string{"cli_test", "serve", cfgfile.Name()}
go func() {
err = cli.NewServerRootCmd().Execute()
if err != nil {
panic(err)
}
}()
WaitTillServerReady(baseURL)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
return string(data), nil
}

28
pkg/extensions/sync/sync_test.go
View file

@ -12,6 +12,7 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"os/exec"
@ -3374,6 +3375,7 @@ func TestSyncOnlyDiff(t *testing.T) {
destConfig.Extensions = &extconf.ExtensionConfig{}
destConfig.Extensions.Search = nil
destConfig.Extensions.Sync = syncConfig
destConfig.Log.Output = path.Join(destDir, "sync.log")
dctlr := api.NewController(destConfig)
@ -3390,32 +3392,18 @@ func TestSyncOnlyDiff(t *testing.T) {
dctlr.Shutdown()
}()
// watch .sync subdir, shouldn't be populated
done := make(chan bool)
var isPopulated bool
go func() {
for {
select {
case <-done:
return
default:
fileList, _ := os.ReadDir(path.Join(destDir, testImage, ".sync"))
if len(fileList) > 0 {
isPopulated = true
}
time.Sleep(200 * time.Millisecond)
}
}
}()
resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
time.Sleep(3 * time.Second)
done <- true
So(isPopulated, ShouldBeFalse)
body, err := ioutil.ReadFile(path.Join(destDir, "sync.log"))
if err != nil {
log.Fatalf("unable to read file: %v", err)
}
So(string(body), ShouldContainSubstring, "already synced image")
})
}