From ee5c842c7d5abb5613b536deb749296beb078bb9 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Tue, 27 Oct 2015 00:07:22 -0600 Subject: [PATCH 1/4] Code to convert between JSON and Caddyfile This will be used by the API so clients have an easier time manipulating the configuration --- caddy/caddyfile/json.go | 153 +++++++++++++++++++++++++++++++++++ caddy/caddyfile/json_test.go | 62 ++++++++++++++ caddy/config.go | 2 +- caddy/parse/parse.go | 8 +- caddy/parse/parsing.go | 11 ++- caddy/parse/parsing_test.go | 2 +- 6 files changed, 229 insertions(+), 9 deletions(-) create mode 100644 caddy/caddyfile/json.go create mode 100644 caddy/caddyfile/json_test.go diff --git a/caddy/caddyfile/json.go b/caddy/caddyfile/json.go new file mode 100644 index 00000000..20b36bcc --- /dev/null +++ b/caddy/caddyfile/json.go @@ -0,0 +1,153 @@ +package caddyfile + +import ( + "bytes" + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/mholt/caddy/caddy/parse" +) + +const filename = "Caddyfile" + +// ToJSON converts caddyfile to its JSON representation. +func ToJSON(caddyfile []byte) ([]byte, error) { + var j Caddyfile + + serverBlocks, err := parse.ServerBlocks(filename, bytes.NewReader(caddyfile), false) + if err != nil { + return nil, err + } + + for _, sb := range serverBlocks { + block := ServerBlock{Body: make(map[string]interface{})} + + for _, host := range sb.HostList() { + block.Hosts = append(block.Hosts, host) + } + + for dir, tokens := range sb.Tokens { + disp := parse.NewDispenserTokens(filename, tokens) + disp.Next() // the first token is the directive; skip it + block.Body[dir] = constructLine(disp) + } + + j = append(j, block) + } + + result, err := json.Marshal(j) + if err != nil { + return nil, err + } + + return result, nil +} + +// constructLine transforms tokens into a JSON-encodable structure; +// but only one line at a time, to be used at the top-level of +// a server block only (where the first token on each line is a +// directive) - not to be used at any other nesting level. +func constructLine(d parse.Dispenser) interface{} { + var args []interface{} + + all := d.RemainingArgs() + for _, arg := range all { + args = append(args, arg) + } + + d.Next() + if d.Val() == "{" { + args = append(args, constructBlock(d)) + } + + return args +} + +// constructBlock recursively processes tokens into a +// JSON-encodable structure. +func constructBlock(d parse.Dispenser) interface{} { + block := make(map[string]interface{}) + + for d.Next() { + if d.Val() == "}" { + break + } + + dir := d.Val() + all := d.RemainingArgs() + + var args []interface{} + for _, arg := range all { + args = append(args, arg) + } + if d.Val() == "{" { + args = append(args, constructBlock(d)) + } + + block[dir] = args + } + + return block +} + +// FromJSON converts JSON-encoded jsonBytes to Caddyfile text +func FromJSON(jsonBytes []byte) ([]byte, error) { + var j Caddyfile + var result string + + err := json.Unmarshal(jsonBytes, &j) + if err != nil { + return nil, err + } + + for _, sb := range j { + for i, host := range sb.Hosts { + if i > 0 { + result += ", " + } + result += host + } + result += jsonToText(sb.Body, 1) + } + + return []byte(result), nil +} + +// jsonToText recursively transforms a scope of JSON into plain +// Caddyfile text. +func jsonToText(scope interface{}, depth int) string { + var result string + + switch val := scope.(type) { + case string: + result += " " + val + case int: + result += " " + strconv.Itoa(val) + case float64: + result += " " + fmt.Sprintf("%f", val) + case bool: + result += " " + fmt.Sprintf("%t", val) + case map[string]interface{}: + result += " {\n" + for param, args := range val { + result += strings.Repeat("\t", depth) + param + result += jsonToText(args, depth+1) + "\n" + } + result += strings.Repeat("\t", depth-1) + "}" + case []interface{}: + for _, v := range val { + result += jsonToText(v, depth) + } + } + + return result +} + +type Caddyfile []ServerBlock + +type ServerBlock struct { + Hosts []string `json:"hosts"` + Body map[string]interface{} `json:"body"` +} diff --git a/caddy/caddyfile/json_test.go b/caddy/caddyfile/json_test.go new file mode 100644 index 00000000..2d94d604 --- /dev/null +++ b/caddy/caddyfile/json_test.go @@ -0,0 +1,62 @@ +package caddyfile + +import "testing" + +var tests = []struct { + caddyfile, json string +}{ + { // 0 + caddyfile: `foo: { + root /bar +}`, + json: `[{"hosts":["foo:"],"body":{"root":["/bar"]}}]`, + }, + { // 1 + caddyfile: `host1:, host2: { + dir { + def + } +}`, + json: `[{"hosts":["host1:","host2:"],"body":{"dir":[{"def":null}]}}]`, + }, + { // 2 + caddyfile: `host1:, host2: { + dir abc { + def ghi + jklmnop + } +}`, + json: `[{"hosts":["host1:","host2:"],"body":{"dir":["abc",{"def":["ghi"],"jklmnop":null}]}}]`, + }, + { // 3 + caddyfile: `host1:1234, host2:5678 { + dir abc { + } +}`, + json: `[{"hosts":["host1:1234","host2:5678"],"body":{"dir":["abc",{}]}}]`, + }, +} + +func TestToJSON(t *testing.T) { + for i, test := range tests { + output, err := ToJSON([]byte(test.caddyfile)) + if err != nil { + t.Errorf("Test %d: %v", i, err) + } + if string(output) != test.json { + t.Errorf("Test %d\nExpected:\n'%s'\nActual:\n'%s'", i, test.json, string(output)) + } + } +} + +func TestFromJSON(t *testing.T) { + for i, test := range tests { + output, err := FromJSON([]byte(test.json)) + if err != nil { + t.Errorf("Test %d: %v", i, err) + } + if string(output) != test.caddyfile { + t.Errorf("Test %d\nExpected:\n'%s'\nActual:\n'%s'", i, test.caddyfile, string(output)) + } + } +} diff --git a/caddy/config.go b/caddy/config.go index dac65784..5688a6db 100644 --- a/caddy/config.go +++ b/caddy/config.go @@ -29,7 +29,7 @@ func Load(filename string, input io.Reader) (Group, error) { flags := log.Flags() log.SetFlags(0) - serverBlocks, err := parse.ServerBlocks(filename, input) + serverBlocks, err := parse.ServerBlocks(filename, input, true) if err != nil { return nil, err } diff --git a/caddy/parse/parse.go b/caddy/parse/parse.go index b44041d4..84043e60 100644 --- a/caddy/parse/parse.go +++ b/caddy/parse/parse.go @@ -5,9 +5,11 @@ import "io" // ServerBlocks parses the input just enough to organize tokens, // in order, by server block. No further parsing is performed. -// Server blocks are returned in the order in which they appear. -func ServerBlocks(filename string, input io.Reader) ([]serverBlock, error) { - p := parser{Dispenser: NewDispenser(filename, input)} +// If checkDirectives is true, only valid directives will be allowed +// otherwise we consider it a parse error. Server blocks are returned +// in the order in which they appear. +func ServerBlocks(filename string, input io.Reader, checkDirectives bool) ([]serverBlock, error) { + p := parser{Dispenser: NewDispenser(filename, input), checkDirectives: checkDirectives} blocks, err := p.parseAll() return blocks, err } diff --git a/caddy/parse/parsing.go b/caddy/parse/parsing.go index 59455391..b24b46ab 100644 --- a/caddy/parse/parsing.go +++ b/caddy/parse/parsing.go @@ -9,8 +9,9 @@ import ( type parser struct { Dispenser - block serverBlock // current server block being parsed - eof bool // if we encounter a valid EOF in a hard place + block serverBlock // current server block being parsed + eof bool // if we encounter a valid EOF in a hard place + checkDirectives bool // if true, directives must be known } func (p *parser) parseAll() ([]serverBlock, error) { @@ -220,8 +221,10 @@ func (p *parser) directive() error { dir := p.Val() nesting := 0 - if _, ok := ValidDirectives[dir]; !ok { - return p.Errf("Unknown directive '%s'", dir) + if p.checkDirectives { + if _, ok := ValidDirectives[dir]; !ok { + return p.Errf("Unknown directive '%s'", dir) + } } // The directive itself is appended as a relevant token diff --git a/caddy/parse/parsing_test.go b/caddy/parse/parsing_test.go index c8a7ef0b..afd5870f 100644 --- a/caddy/parse/parsing_test.go +++ b/caddy/parse/parsing_test.go @@ -375,6 +375,6 @@ func setupParseTests() { func testParser(input string) parser { buf := strings.NewReader(input) - p := parser{Dispenser: NewDispenser("Test", buf)} + p := parser{Dispenser: NewDispenser("Test", buf), checkDirectives: true} return p } From 0f19df8a81e01bb53bffa16833bf816ea3a4f14d Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Tue, 27 Oct 2015 00:43:24 -0600 Subject: [PATCH 2/4] Keep tests deterministic --- caddy/caddyfile/json_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/caddy/caddyfile/json_test.go b/caddy/caddyfile/json_test.go index 2d94d604..11e1b1f4 100644 --- a/caddy/caddyfile/json_test.go +++ b/caddy/caddyfile/json_test.go @@ -23,10 +23,9 @@ var tests = []struct { caddyfile: `host1:, host2: { dir abc { def ghi - jklmnop } }`, - json: `[{"hosts":["host1:","host2:"],"body":{"dir":["abc",{"def":["ghi"],"jklmnop":null}]}}]`, + json: `[{"hosts":["host1:","host2:"],"body":{"dir":["abc",{"def":["ghi"]}]}}]`, }, { // 3 caddyfile: `host1:1234, host2:5678 { From 976f5182e1694f43c3ee4cca7b503402b1049ea6 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 29 Oct 2015 00:22:56 -0600 Subject: [PATCH 3/4] caddyfile: Better string and number handling --- caddy/caddyfile/json.go | 12 ++++++++---- caddy/caddyfile/json_test.go | 37 ++++++++++++++++++++++++++++++------ 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/caddy/caddyfile/json.go b/caddy/caddyfile/json.go index 20b36bcc..4617ec03 100644 --- a/caddy/caddyfile/json.go +++ b/caddy/caddyfile/json.go @@ -25,7 +25,7 @@ func ToJSON(caddyfile []byte) ([]byte, error) { block := ServerBlock{Body: make(map[string]interface{})} for _, host := range sb.HostList() { - block.Hosts = append(block.Hosts, host) + block.Hosts = append(block.Hosts, strings.TrimSuffix(host, ":")) } for dir, tokens := range sb.Tokens { @@ -107,7 +107,7 @@ func FromJSON(jsonBytes []byte) ([]byte, error) { if i > 0 { result += ", " } - result += host + result += strings.TrimSuffix(host, ":") } result += jsonToText(sb.Body, 1) } @@ -122,11 +122,15 @@ func jsonToText(scope interface{}, depth int) string { switch val := scope.(type) { case string: - result += " " + val + if strings.ContainsAny(val, "\" \n\t\r") { + result += ` "` + strings.Replace(val, "\"", "\\\"", -1) + `"` + } else { + result += " " + val + } case int: result += " " + strconv.Itoa(val) case float64: - result += " " + fmt.Sprintf("%f", val) + result += " " + fmt.Sprintf("%v", val) case bool: result += " " + fmt.Sprintf("%t", val) case map[string]interface{}: diff --git a/caddy/caddyfile/json_test.go b/caddy/caddyfile/json_test.go index 11e1b1f4..2e5ed445 100644 --- a/caddy/caddyfile/json_test.go +++ b/caddy/caddyfile/json_test.go @@ -6,26 +6,26 @@ var tests = []struct { caddyfile, json string }{ { // 0 - caddyfile: `foo: { + caddyfile: `foo { root /bar }`, - json: `[{"hosts":["foo:"],"body":{"root":["/bar"]}}]`, + json: `[{"hosts":["foo"],"body":{"root":["/bar"]}}]`, }, { // 1 - caddyfile: `host1:, host2: { + caddyfile: `host1, host2 { dir { def } }`, - json: `[{"hosts":["host1:","host2:"],"body":{"dir":[{"def":null}]}}]`, + json: `[{"hosts":["host1","host2"],"body":{"dir":[{"def":null}]}}]`, }, { // 2 - caddyfile: `host1:, host2: { + caddyfile: `host1, host2 { dir abc { def ghi } }`, - json: `[{"hosts":["host1:","host2:"],"body":{"dir":["abc",{"def":["ghi"]}]}}]`, + json: `[{"hosts":["host1","host2"],"body":{"dir":["abc",{"def":["ghi"]}]}}]`, }, { // 3 caddyfile: `host1:1234, host2:5678 { @@ -34,6 +34,31 @@ var tests = []struct { }`, json: `[{"hosts":["host1:1234","host2:5678"],"body":{"dir":["abc",{}]}}]`, }, + { // 4 + caddyfile: `host { + foo "bar baz" +}`, + json: `[{"hosts":["host"],"body":{"foo":["bar baz"]}}]`, + }, + { // 5 + caddyfile: `host, host:80 { + foo "bar \"baz\"" +}`, + json: `[{"hosts":["host","host:80"],"body":{"foo":["bar \"baz\""]}}]`, + }, + { // 6 + caddyfile: `host { + foo "bar +baz" +}`, + json: `[{"hosts":["host"],"body":{"foo":["bar\nbaz"]}}]`, + }, + { // 7 + caddyfile: `host { + dir 123 4.56 true +}`, + json: `[{"hosts":["host"],"body":{"dir":["123","4.56","true"]}}]`, // NOTE: I guess we assume numbers and booleans should be encoded as strings...? + }, } func TestToJSON(t *testing.T) { From efeeece73543c5ec7d94a5eb942583b366775c2a Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 29 Oct 2015 10:13:30 -0600 Subject: [PATCH 4/4] caddyfile: http and https hosts should render in URL format --- caddy/caddyfile/json.go | 6 ++++++ caddy/caddyfile/json_test.go | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/caddy/caddyfile/json.go b/caddy/caddyfile/json.go index 4617ec03..42171e7a 100644 --- a/caddy/caddyfile/json.go +++ b/caddy/caddyfile/json.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "net" "strconv" "strings" @@ -104,6 +105,11 @@ func FromJSON(jsonBytes []byte) ([]byte, error) { for _, sb := range j { for i, host := range sb.Hosts { + if hostname, port, err := net.SplitHostPort(host); err == nil { + if port == "http" || port == "https" { + host = port + "://" + hostname + } + } if i > 0 { result += ", " } diff --git a/caddy/caddyfile/json_test.go b/caddy/caddyfile/json_test.go index 2e5ed445..44abf982 100644 --- a/caddy/caddyfile/json_test.go +++ b/caddy/caddyfile/json_test.go @@ -59,6 +59,11 @@ baz" }`, json: `[{"hosts":["host"],"body":{"dir":["123","4.56","true"]}}]`, // NOTE: I guess we assume numbers and booleans should be encoded as strings...? }, + { // 8 + caddyfile: `http://host, https://host { +}`, + json: `[{"hosts":["host:http","host:https"],"body":{}}]`, // hosts in JSON are always host:port format (if port is specified) + }, } func TestToJSON(t *testing.T) {