From fbc18c5b8560508920efaec89fa4967128904422 Mon Sep 17 00:00:00 2001 From: Tw Date: Sun, 22 Nov 2015 10:50:31 +0800 Subject: [PATCH 01/44] markdown: fix json format parse issue We can't use json meta parser's remaining buffered data as the markdown body because it may not contain the entire original content. Now we adopt the way like toml and yaml parser's way to extract the meta content at first. Also when spilting the meta data and content body, additional io.Copy is unnecessary. Fix issue #355 Signed-off-by: Tw --- middleware/markdown/metadata.go | 71 ++++++++++++--------------------- 1 file changed, 26 insertions(+), 45 deletions(-) diff --git a/middleware/markdown/metadata.go b/middleware/markdown/metadata.go index 103b97b2..2a7e434f 100644 --- a/middleware/markdown/metadata.go +++ b/middleware/markdown/metadata.go @@ -1,11 +1,9 @@ package markdown import ( - "bufio" "bytes" "encoding/json" "fmt" - "io" "github.com/BurntSushi/toml" "gopkg.in/yaml.v2" @@ -73,23 +71,20 @@ type JSONMetadataParser struct { // Parse the metadata func (j *JSONMetadataParser) Parse(b []byte) ([]byte, error) { + b, markdown, err := extractMetadata(j, b) + if err != nil { + return markdown, err + } m := make(map[string]interface{}) // Read the preceding JSON object decoder := json.NewDecoder(bytes.NewReader(b)) if err := decoder.Decode(&m); err != nil { - return b, err + return markdown, err } j.metadata.load(m) - // Retrieve remaining bytes after decoding - buf := make([]byte, len(b)) - n, err := decoder.Buffered().Read(buf) - if err != nil { - return b, err - } - - return buf[:n], nil + return markdown, nil } // Metadata returns parsed metadata. It should be called @@ -183,43 +178,29 @@ func (y *YAMLMetadataParser) Closing() []byte { // It returns the metadata, the remaining bytes (markdown), and an error, if any. func extractMetadata(parser MetadataParser, b []byte) (metadata []byte, markdown []byte, err error) { b = bytes.TrimSpace(b) - reader := bufio.NewReader(bytes.NewBuffer(b)) - - // Read first line, which should indicate metadata or not - line, err := reader.ReadBytes('\n') - if err != nil || !bytes.Equal(bytes.TrimSpace(line), parser.Opening()) { + openingLine := append(parser.Opening(), '\n') + closingLine := append(parser.Closing(), '\n') + if !bytes.HasPrefix(b, openingLine) { return nil, b, fmt.Errorf("first line missing expected metadata identifier") } - - // buffer for metadata contents - metaBuf := bytes.Buffer{} - - // Read remaining lines until closing identifier is found - for { - line, err := reader.ReadBytes('\n') - if err != nil && err != io.EOF { - return nil, nil, err - } - - // if closing identifier found, the remaining bytes must be markdown content - if bytes.Equal(bytes.TrimSpace(line), parser.Closing()) { - break - } - - // if file ended, by this point no closing identifier was found - if err == io.EOF { - return nil, nil, fmt.Errorf("metadata not closed ('%s' not found)", parser.Closing()) - } - - metaBuf.Write(line) - metaBuf.WriteString("\r\n") + metaStart := len(openingLine) + if _, ok := parser.(*JSONMetadataParser); ok { + metaStart = 0 } - - // By now, the rest of the buffer contains markdown content - contentBuf := new(bytes.Buffer) - io.Copy(contentBuf, reader) - - return metaBuf.Bytes(), contentBuf.Bytes(), nil + metaEnd := bytes.Index(b[metaStart:], closingLine) + if metaEnd == -1 { + return nil, nil, fmt.Errorf("metadata not closed ('%s' not found)", parser.Closing()) + } + metaEnd += metaStart + if _, ok := parser.(*JSONMetadataParser); ok { + metaEnd += len(closingLine) + } + metadata = b[metaStart:metaEnd] + markdown = b[metaEnd:] + if _, ok := parser.(*JSONMetadataParser); !ok { + markdown = b[metaEnd+len(closingLine):] + } + return metadata, markdown, nil } // findParser finds the parser using line that contains opening identifier From 19c6bbf6a2275dab38a6307dc84b45c9a52fd27a Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 2 Dec 2015 11:42:50 -0700 Subject: [PATCH 02/44] Update changelist (env vars) --- dist/CHANGES.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dist/CHANGES.txt b/dist/CHANGES.txt index e98535b5..01657feb 100644 --- a/dist/CHANGES.txt +++ b/dist/CHANGES.txt @@ -14,6 +14,7 @@ CHANGES - New -grace flag to customize the graceful shutdown timeout - New support for SIGHUP, SIGTERM, and SIGQUIT signals - browse: Render filenames with multiple whitespace properly +- core: Use environment variables in Caddyfile - markdown: Include Last-Modified header in response - markdown: Render tables, strikethrough, and fenced code blocks - proxy: Ability to exclude/ignore paths from proxying From 9002db2ae03906fdbfd3ede2fc32825388889ee9 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 2 Dec 2015 12:40:57 -0700 Subject: [PATCH 03/44] letsencrypt: Remote duplicate hosts from certificate request Domain names must be unique in cert bundle request or really bad things happen (like, um, a panic) --- caddy/letsencrypt/letsencrypt.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/caddy/letsencrypt/letsencrypt.go b/caddy/letsencrypt/letsencrypt.go index 16312bd6..feeac297 100644 --- a/caddy/letsencrypt/letsencrypt.go +++ b/caddy/letsencrypt/letsencrypt.go @@ -79,9 +79,19 @@ func Activate(configs []server.Config) ([]server.Config, error) { } // little bit of housekeeping; gather the hostnames into a slice - hosts := make([]string, len(cfgIndexes)) - for i, idx := range cfgIndexes { - hosts[i] = configs[idx].Host + var hosts []string + for _, idx := range cfgIndexes { + // don't allow duplicates (happens when serving same host on multiple ports!) + var duplicate bool + for _, otherHost := range hosts { + if configs[idx].Host == otherHost { + duplicate = true + break + } + } + if !duplicate { + hosts = append(hosts, configs[idx].Host) + } } // client is ready, so let's get free, trusted SSL certificates! From 6f4835f91a4c18a4acd7c5d8977761b8fdf71b0f Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Thu, 3 Dec 2015 00:41:12 +0100 Subject: [PATCH 04/44] Markdown: Fix "metadata not closed" bug. More tests. --- middleware/markdown/metadata.go | 6 +- middleware/markdown/metadata_test.go | 172 +++++++++++++++++---------- 2 files changed, 109 insertions(+), 69 deletions(-) diff --git a/middleware/markdown/metadata.go b/middleware/markdown/metadata.go index 2a7e434f..07c00801 100644 --- a/middleware/markdown/metadata.go +++ b/middleware/markdown/metadata.go @@ -4,10 +4,10 @@ import ( "bytes" "encoding/json" "fmt" + "time" "github.com/BurntSushi/toml" "gopkg.in/yaml.v2" - "time" ) // Metadata stores a page's metadata @@ -178,8 +178,8 @@ func (y *YAMLMetadataParser) Closing() []byte { // It returns the metadata, the remaining bytes (markdown), and an error, if any. func extractMetadata(parser MetadataParser, b []byte) (metadata []byte, markdown []byte, err error) { b = bytes.TrimSpace(b) - openingLine := append(parser.Opening(), '\n') - closingLine := append(parser.Closing(), '\n') + openingLine := parser.Opening() + closingLine := parser.Closing() if !bytes.HasPrefix(b, openingLine) { return nil, b, fmt.Errorf("first line missing expected metadata identifier") } diff --git a/middleware/markdown/metadata_test.go b/middleware/markdown/metadata_test.go index b93c9ddc..b8067016 100644 --- a/middleware/markdown/metadata_test.go +++ b/middleware/markdown/metadata_test.go @@ -8,72 +8,6 @@ import ( "testing" ) -var TOML = [4]string{` -title = "A title" -template = "default" -name = "value" -`, - `+++ -title = "A title" -template = "default" -name = "value" -+++ -Page content -`, - `+++ -title = "A title" -template = "default" -name = "value" - `, - `title = "A title" template = "default" [variables] name = "value"`, -} - -var YAML = [4]string{` -title : A title -template : default -name : value -`, - `--- -title : A title -template : default -name : value ---- -Page content -`, - `--- -title : A title -template : default -name : value -`, - `title : A title template : default variables : name : value`, -} -var JSON = [4]string{` - "title" : "A title", - "template" : "default", - "name" : "value" -`, - `{ - "title" : "A title", - "template" : "default", - "name" : "value" -} -Page content -`, - ` -{ - "title" : "A title", - "template" : "default", - "name" : "value" -`, - ` -{{ - "title" : "A title", - "template" : "default", - "name" : "value" -} - `, -} - func check(t *testing.T, err error) { if err != nil { t.Fatal(err) @@ -81,6 +15,71 @@ func check(t *testing.T, err error) { } func TestParsers(t *testing.T) { + var TOML = [4]string{` +title = "A title" +template = "default" +name = "value" +`, + `+++ +title = "A title" +template = "default" +name = "value" ++++ +Page content + `, + `+++ +title = "A title" +template = "default" +name = "value" + `, + `title = "A title" template = "default" [variables] name = "value"`, + } + + var YAML = [4]string{` +title : A title +template : default +name : value +`, + `--- +title : A title +template : default +name : value +--- + Page content + `, + `--- +title : A title +template : default +name : value + `, + `title : A title template : default variables : name : value`, + } + var JSON = [4]string{` +"title" : "A title", +"template" : "default", +"name" : "value" +`, + `{ + "title" : "A title", + "template" : "default", + "name" : "value" +} +Page content + `, + ` +{ + "title" : "A title", + "template" : "default", + "name" : "value" + `, + ` +{{ + "title" : "A title", + "template" : "default", + "name" : "value" +} + `, + } expected := Metadata{ Title: "A title", Template: "default", @@ -153,3 +152,44 @@ func TestParsers(t *testing.T) { } } + +func TestLargeBody(t *testing.T) { + var JSON = `{ +"template": "chapter" +} + +Mycket olika byggnader har man i de nordiska rikena: pyramidformiga, kilformiga, välvda, runda och fyrkantiga. De pyramidformiga består helt enkelt av träribbor, som upptill löper samman och nedtill bildar en vidare krets; de är avsedda att användas av hantverkarna under sommaren, för att de inte ska plågas av solen, på samma gång som de besväras av rök och eld. De kilformiga husen är i regel försedda med höga tak, för att de täta och tunga snömassorna fortare ska kunna blåsa av och inte tynga ned taken. Dessa är täckta av björknäver, tegel eller kluvet spån av furu - för kådans skull -, gran, ek eller bok; taken på de förmögnas hus däremot med plåtar av koppar eller bly, i likhet med kyrktaken. Valvbyggnaderna uppförs ganska konstnärligt till skydd mot våldsamma vindar och snöfall, görs av sten eller trä, och är avsedda för olika alldagliga viktiga ändamål. Liknande byggnader kan finnas i stormännens gårdar där de används som förvaringsrum för husgeråd och jordbruksredskap. De runda byggnaderna - som för övrigt är de högst sällsynta - används av konstnärer, som vid sitt arbete behöver ett jämnt fördelat ljus från taket. Vanligast är de fyrkantiga husen, vars grova bjälkar är synnerligen väl hopfogade i hörnen - ett sant mästerverk av byggnadskonst; även dessa har fönster högt uppe i taken, för att dagsljuset skall kunna strömma in och ge alla därinne full belysning. Stenhusen har dörröppningar i förhållande till byggnadens storlek, men smala fönstergluggar, som skydd mot den stränga kölden, frosten och snön. Vore de större och vidare, såsom fönstren i Italien, skulle husen i följd av den fint yrande snön, som röres upp av den starka blåsten, precis som dammet av virvelvinden, snart nog fyllas med massor av snö och inte kunna stå emot dess tryck, utan störta samman. + + ` + var TOML = `+++ +template = "chapter" ++++ + +Mycket olika byggnader har man i de nordiska rikena: pyramidformiga, kilformiga, välvda, runda och fyrkantiga. De pyramidformiga består helt enkelt av träribbor, som upptill löper samman och nedtill bildar en vidare krets; de är avsedda att användas av hantverkarna under sommaren, för att de inte ska plågas av solen, på samma gång som de besväras av rök och eld. De kilformiga husen är i regel försedda med höga tak, för att de täta och tunga snömassorna fortare ska kunna blåsa av och inte tynga ned taken. Dessa är täckta av björknäver, tegel eller kluvet spån av furu - för kådans skull -, gran, ek eller bok; taken på de förmögnas hus däremot med plåtar av koppar eller bly, i likhet med kyrktaken. Valvbyggnaderna uppförs ganska konstnärligt till skydd mot våldsamma vindar och snöfall, görs av sten eller trä, och är avsedda för olika alldagliga viktiga ändamål. Liknande byggnader kan finnas i stormännens gårdar där de används som förvaringsrum för husgeråd och jordbruksredskap. De runda byggnaderna - som för övrigt är de högst sällsynta - används av konstnärer, som vid sitt arbete behöver ett jämnt fördelat ljus från taket. Vanligast är de fyrkantiga husen, vars grova bjälkar är synnerligen väl hopfogade i hörnen - ett sant mästerverk av byggnadskonst; även dessa har fönster högt uppe i taken, för att dagsljuset skall kunna strömma in och ge alla därinne full belysning. Stenhusen har dörröppningar i förhållande till byggnadens storlek, men smala fönstergluggar, som skydd mot den stränga kölden, frosten och snön. Vore de större och vidare, såsom fönstren i Italien, skulle husen i följd av den fint yrande snön, som röres upp av den starka blåsten, precis som dammet av virvelvinden, snart nog fyllas med massor av snö och inte kunna stå emot dess tryck, utan störta samman. + + ` + var YAML = `--- +template : chapter +--- + +Mycket olika byggnader har man i de nordiska rikena: pyramidformiga, kilformiga, välvda, runda och fyrkantiga. De pyramidformiga består helt enkelt av träribbor, som upptill löper samman och nedtill bildar en vidare krets; de är avsedda att användas av hantverkarna under sommaren, för att de inte ska plågas av solen, på samma gång som de besväras av rök och eld. De kilformiga husen är i regel försedda med höga tak, för att de täta och tunga snömassorna fortare ska kunna blåsa av och inte tynga ned taken. Dessa är täckta av björknäver, tegel eller kluvet spån av furu - för kådans skull -, gran, ek eller bok; taken på de förmögnas hus däremot med plåtar av koppar eller bly, i likhet med kyrktaken. Valvbyggnaderna uppförs ganska konstnärligt till skydd mot våldsamma vindar och snöfall, görs av sten eller trä, och är avsedda för olika alldagliga viktiga ändamål. Liknande byggnader kan finnas i stormännens gårdar där de används som förvaringsrum för husgeråd och jordbruksredskap. De runda byggnaderna - som för övrigt är de högst sällsynta - används av konstnärer, som vid sitt arbete behöver ett jämnt fördelat ljus från taket. Vanligast är de fyrkantiga husen, vars grova bjälkar är synnerligen väl hopfogade i hörnen - ett sant mästerverk av byggnadskonst; även dessa har fönster högt uppe i taken, för att dagsljuset skall kunna strömma in och ge alla därinne full belysning. Stenhusen har dörröppningar i förhållande till byggnadens storlek, men smala fönstergluggar, som skydd mot den stränga kölden, frosten och snön. Vore de större och vidare, såsom fönstren i Italien, skulle husen i följd av den fint yrande snön, som röres upp av den starka blåsten, precis som dammet av virvelvinden, snart nog fyllas med massor av snö och inte kunna stå emot dess tryck, utan störta samman. + + ` + var expectedBody = `Mycket olika byggnader har man i de nordiska rikena: pyramidformiga, kilformiga, välvda, runda och fyrkantiga. De pyramidformiga består helt enkelt av träribbor, som upptill löper samman och nedtill bildar en vidare krets; de är avsedda att användas av hantverkarna under sommaren, för att de inte ska plågas av solen, på samma gång som de besväras av rök och eld. De kilformiga husen är i regel försedda med höga tak, för att de täta och tunga snömassorna fortare ska kunna blåsa av och inte tynga ned taken. Dessa är täckta av björknäver, tegel eller kluvet spån av furu - för kådans skull -, gran, ek eller bok; taken på de förmögnas hus däremot med plåtar av koppar eller bly, i likhet med kyrktaken. Valvbyggnaderna uppförs ganska konstnärligt till skydd mot våldsamma vindar och snöfall, görs av sten eller trä, och är avsedda för olika alldagliga viktiga ändamål. Liknande byggnader kan finnas i stormännens gårdar där de används som förvaringsrum för husgeråd och jordbruksredskap. De runda byggnaderna - som för övrigt är de högst sällsynta - används av konstnärer, som vid sitt arbete behöver ett jämnt fördelat ljus från taket. Vanligast är de fyrkantiga husen, vars grova bjälkar är synnerligen väl hopfogade i hörnen - ett sant mästerverk av byggnadskonst; även dessa har fönster högt uppe i taken, för att dagsljuset skall kunna strömma in och ge alla därinne full belysning. Stenhusen har dörröppningar i förhållande till byggnadens storlek, men smala fönstergluggar, som skydd mot den stränga kölden, frosten och snön. Vore de större och vidare, såsom fönstren i Italien, skulle husen i följd av den fint yrande snön, som röres upp av den starka blåsten, precis som dammet av virvelvinden, snart nog fyllas med massor av snö och inte kunna stå emot dess tryck, utan störta samman. +` + data := []struct { + parser MetadataParser + testData string + name string + }{ + {&JSONMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, JSON, "json"}, + {&YAMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, YAML, "yaml"}, + {&TOMLMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, TOML, "toml"}, + } + for _, v := range data { + // metadata without identifiers + if md, err := v.parser.Parse([]byte(v.testData)); err != nil || strings.TrimSpace(string(md)) != strings.TrimSpace(expectedBody) { + t.Fatalf("Error not expected and/or markdown not equal for %v", v.name) + } + } +} From fd14f257df1b1e1862c24705628cb79ab4a68a8e Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 2 Dec 2015 17:04:58 -0700 Subject: [PATCH 05/44] markdown: Add (currently failing) test for empty body --- middleware/markdown/metadata_test.go | 69 +++++++++++++++++++--------- 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/middleware/markdown/metadata_test.go b/middleware/markdown/metadata_test.go index b8067016..285caf97 100644 --- a/middleware/markdown/metadata_test.go +++ b/middleware/markdown/metadata_test.go @@ -14,72 +14,92 @@ func check(t *testing.T, err error) { } } -func TestParsers(t *testing.T) { - var TOML = [4]string{` +var TOML = [5]string{` title = "A title" template = "default" name = "value" `, - `+++ + `+++ title = "A title" template = "default" name = "value" +++ Page content `, - `+++ + `+++ title = "A title" template = "default" name = "value" `, - `title = "A title" template = "default" [variables] name = "value"`, - } + `title = "A title" template = "default" [variables] name = "value"`, + `+++ +title = "A title" +template = "default" +name = "value" ++++ +`, +} - var YAML = [4]string{` +var YAML = [5]string{` title : A title template : default name : value `, - `--- + `--- title : A title template : default name : value --- Page content `, - `--- + `--- title : A title template : default name : value `, - `title : A title template : default variables : name : value`, - } - var JSON = [4]string{` -"title" : "A title", -"template" : "default", -"name" : "value" + `title : A title template : default variables : name : value`, + `--- +title : A title +template : default +name : value +--- `, - `{ +} + +var JSON = [5]string{` + "title" : "A title", + "template" : "default", + "name" : "value" +`, + `{ "title" : "A title", "template" : "default", "name" : "value" } Page content `, - ` + ` { "title" : "A title", "template" : "default", "name" : "value" `, - ` -{{ + ` +{ + "title" :: "A title", + "template" : "default", + "name" : "value" +} + `, + `{ "title" : "A title", "template" : "default", "name" : "value" } - `, - } +`, +} + +func TestParsers(t *testing.T) { expected := Metadata{ Title: "A title", Template: "default", @@ -106,7 +126,7 @@ Page content data := []struct { parser MetadataParser - testData [4]string + testData [5]string name string }{ {&JSONMetadataParser{metadata: Metadata{Variables: make(map[string]string)}}, JSON, "json"}, @@ -149,6 +169,11 @@ Page content if md, err = v.parser.Parse([]byte(v.testData[3])); err == nil { t.Fatalf("Expected error for invalid metadata for %v", v.name) } + + // front matter but no body + if md, err = v.parser.Parse([]byte(v.testData[4])); err != nil { + t.Fatalf("Unexpected error for valid metadata but no body for %v", v.name) + } } } From 5b93799a6230357827335497b7a09f8006ce9b57 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 3 Dec 2015 19:52:15 -0700 Subject: [PATCH 06/44] Version 0.8.0 --- dist/CHANGES.txt | 6 +++--- dist/README.txt | 2 +- main.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dist/CHANGES.txt b/dist/CHANGES.txt index 01657feb..97542ec7 100644 --- a/dist/CHANGES.txt +++ b/dist/CHANGES.txt @@ -1,8 +1,8 @@ CHANGES -0.8 beta -- Let's Encrypt (free, automatic, fully-managed HTTPS for your sites) -- Graceful restarts (for POSIX-compatible systems) +0.8.0 (December 4, 2015) +- HTTPS by default via Let's Encrypt (certs & keys are fully managed) +- Graceful restarts (on POSIX-compliant systems) - Major internal refactoring to allow use of Caddy as library - New directive 'mime' to customize Content-Type based on file extension - New -accept flag to accept Let's Encrypt SA without prompt diff --git a/dist/README.txt b/dist/README.txt index 0fd9d1a0..aa0a0e78 100644 --- a/dist/README.txt +++ b/dist/README.txt @@ -1,4 +1,4 @@ -CADDY 0.8 beta 4 +CADDY 0.8 Website https://caddyserver.com diff --git a/main.go b/main.go index b08d101a..f0055d42 100644 --- a/main.go +++ b/main.go @@ -26,13 +26,13 @@ var ( const ( appName = "Caddy" - appVersion = "0.8 beta 4" + appVersion = "0.8" ) func init() { caddy.TrapSignals() flag.BoolVar(&letsencrypt.Agreed, "agree", false, "Agree to Let's Encrypt Subscriber Agreement") - flag.StringVar(&letsencrypt.CAUrl, "ca", "https://acme-staging.api.letsencrypt.org/directory", "Certificate authority ACME server") + flag.StringVar(&letsencrypt.CAUrl, "ca", "https://acme-v01.api.letsencrypt.org/directory", "Certificate authority ACME server") flag.StringVar(&conf, "conf", "", "Configuration file to use (default="+caddy.DefaultConfigFile+")") flag.StringVar(&cpu, "cpu", "100%", "CPU cap") flag.StringVar(&letsencrypt.DefaultEmail, "email", "", "Default Let's Encrypt account email address") From b3f5e4d4ad08a22900c4f11b641c0aadfcd2eee9 Mon Sep 17 00:00:00 2001 From: ReadmeCritic Date: Fri, 4 Dec 2015 07:20:56 -0800 Subject: [PATCH 07/44] Update README URLs based on HTTP redirects --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 6698dccf..cb81c175 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ You may also be interested in the [developer guide] ## Running from Source -Note: You will need **[Go 1.4](https://golang.org/dl)** or a later version. +Note: You will need **[Go 1.4](https://golang.org/dl/)** or a later version. 1. `$ go get github.com/mholt/caddy` 2. `cd` into your website's directory @@ -120,11 +120,11 @@ ports < 1024 like 80 and 443. Caddy is available as a Docker container from any of these sources: -- [abiosoft/caddy](https://registry.hub.docker.com/u/abiosoft/caddy/) -- [darron/caddy](https://registry.hub.docker.com/u/darron/caddy/) -- [joshix/caddy](https://registry.hub.docker.com/u/joshix/caddy/) -- [jumanjiman/caddy](https://registry.hub.docker.com/u/jumanjiman/caddy/) -- [zenithar/nano-caddy](https://registry.hub.docker.com/u/zenithar/nano-caddy/) +- [abiosoft/caddy](https://hub.docker.com/r/abiosoft/caddy/) +- [darron/caddy](https://hub.docker.com/r/darron/caddy/) +- [joshix/caddy](https://hub.docker.com/r/joshix/caddy/) +- [jumanjiman/caddy](https://hub.docker.com/r/jumanjiman/caddy/) +- [zenithar/nano-caddy](https://hub.docker.com/r/zenithar/nano-caddy/) From 12f594779c9b8a23b69aefe5363e1eac087db646 Mon Sep 17 00:00:00 2001 From: Radim Marek Date: Fri, 4 Dec 2015 23:04:12 +0100 Subject: [PATCH 08/44] Added support magic characters in import pattern Import now allows to use the star wildcard, question mark and square brackets as used by filepath.Glob --- caddy/parse/parsing.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/caddy/parse/parsing.go b/caddy/parse/parsing.go index 03d9d800..e71a03f5 100644 --- a/caddy/parse/parsing.go +++ b/caddy/parse/parsing.go @@ -184,11 +184,26 @@ func (p *parser) doImport() error { if !p.NextArg() { return p.ArgErr() } - importFile := p.Val() + importPattern := p.Val() if p.NextArg() { return p.Err("Import allows only one file to import") } + matches, err := filepath.Glob(importPattern) + if err != nil { + return p.Errf("Failed to use import pattern %s - %s", importPattern, err.Error()) + } + + for _, importFile := range matches { + if err := p.doSingleImport(importFile); err != nil { + return err + } + } + + return nil +} + +func (p *parser) doSingleImport(importFile string) error { file, err := os.Open(importFile) if err != nil { return p.Errf("Could not import %s - %v", importFile, err) From d1216f409dd8f7278757be951277758347e66172 Mon Sep 17 00:00:00 2001 From: Radim Marek Date: Fri, 4 Dec 2015 23:21:28 +0100 Subject: [PATCH 09/44] Handle no matches --- caddy/parse/parsing.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/caddy/parse/parsing.go b/caddy/parse/parsing.go index e71a03f5..8aa711c3 100644 --- a/caddy/parse/parsing.go +++ b/caddy/parse/parsing.go @@ -194,6 +194,10 @@ func (p *parser) doImport() error { return p.Errf("Failed to use import pattern %s - %s", importPattern, err.Error()) } + if len(matches) == 0 { + return p.Errf("No files matching the import pattern %s", importPattern) + } + for _, importFile := range matches { if err := p.doSingleImport(importFile); err != nil { return err From e17a18365d4ef83d8865065b6cad772c825a3c83 Mon Sep 17 00:00:00 2001 From: Carsten Hagemann Date: Sat, 5 Dec 2015 13:05:51 +0100 Subject: [PATCH 10/44] Latest Go 1.5 version --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 514e3a1c..19ba6dba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: go go: - 1.4.3 - - 1.5.1 + - 1.5.2 - tip install: From d56a9a1c5dcbfa3191de905884c7016feb168936 Mon Sep 17 00:00:00 2001 From: Radim Marek Date: Sun, 6 Dec 2015 23:49:21 +0100 Subject: [PATCH 11/44] Correct position of the newly imported tokens --- caddy/parse/parsing.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/caddy/parse/parsing.go b/caddy/parse/parsing.go index 8aa711c3..e011fd8c 100644 --- a/caddy/parse/parsing.go +++ b/caddy/parse/parsing.go @@ -198,12 +198,23 @@ func (p *parser) doImport() error { return p.Errf("No files matching the import pattern %s", importPattern) } + // Splice out the import directive and its argument (2 tokens total) + // and insert the imported tokens in their place. + tokensBefore := p.tokens[:p.cursor-1] + tokensAfter := p.tokens[p.cursor+1:] + // cursor was advanced one position to read filename; rewind it + p.cursor-- + + p.tokens = tokensBefore + for _, importFile := range matches { if err := p.doSingleImport(importFile); err != nil { return err } } + p.tokens = append(p.tokens, append(tokensAfter)...) + return nil } @@ -222,10 +233,7 @@ func (p *parser) doSingleImport(importFile string) error { // Splice out the import directive and its argument (2 tokens total) // and insert the imported tokens in their place. - tokensBefore := p.tokens[:p.cursor-1] - tokensAfter := p.tokens[p.cursor+1:] - p.tokens = append(tokensBefore, append(importedTokens, tokensAfter...)...) - p.cursor-- // cursor was advanced one position to read the filename; rewind it + p.tokens = append(p.tokens, append(importedTokens)...) return nil } From ab5087e215271bb9b6d64f84ee7d69194eec2827 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Mon, 7 Dec 2015 23:17:05 +0100 Subject: [PATCH 12/44] Gzip: support for min_length. --- caddy/setup/gzip.go | 30 ++++++-- caddy/setup/gzip_test.go | 12 ++++ middleware/gzip/gzip.go | 20 ++++-- middleware/gzip/gzip_test.go | 2 +- .../gzip/{filter.go => request_filter.go} | 8 +-- ...{filter_test.go => request_filter_test.go} | 4 +- middleware/gzip/response_filter.go | 62 ++++++++++++++++ middleware/gzip/response_filter_test.go | 70 +++++++++++++++++++ 8 files changed, 192 insertions(+), 16 deletions(-) rename middleware/gzip/{filter.go => request_filter.go} (91%) rename middleware/gzip/{filter_test.go => request_filter_test.go} (95%) create mode 100644 middleware/gzip/response_filter.go create mode 100644 middleware/gzip/response_filter_test.go diff --git a/caddy/setup/gzip.go b/caddy/setup/gzip.go index a81c5f17..40c81252 100644 --- a/caddy/setup/gzip.go +++ b/caddy/setup/gzip.go @@ -27,9 +27,13 @@ func gzipParse(c *Controller) ([]gzip.Config, error) { for c.Next() { config := gzip.Config{} + // Request Filters pathFilter := gzip.PathFilter{IgnoredPaths: make(gzip.Set)} extFilter := gzip.ExtFilter{Exts: make(gzip.Set)} + // Response Filters + lengthFilter := gzip.LengthFilter(0) + // No extra args expected if len(c.RemainingArgs()) > 0 { return configs, c.ArgErr() @@ -68,24 +72,42 @@ func gzipParse(c *Controller) ([]gzip.Config, error) { } level, _ := strconv.Atoi(c.Val()) config.Level = level + case "min_length": + if !c.NextArg() { + return configs, c.ArgErr() + } + length, err := strconv.ParseInt(c.Val(), 10, 64) + if err != nil { + return configs, err + } else if length == 0 { + return configs, fmt.Errorf(`gzip: min_length must be greater than 0`) + } + lengthFilter = gzip.LengthFilter(length) default: return configs, c.ArgErr() } } - config.Filters = []gzip.Filter{} + // Request Filters + config.RequestFilters = []gzip.RequestFilter{} // If ignored paths are specified, put in front to filter with path first if len(pathFilter.IgnoredPaths) > 0 { - config.Filters = []gzip.Filter{pathFilter} + config.RequestFilters = []gzip.RequestFilter{pathFilter} } // Then, if extensions are specified, use those to filter. // Otherwise, use default extensions filter. if len(extFilter.Exts) > 0 { - config.Filters = append(config.Filters, extFilter) + config.RequestFilters = append(config.RequestFilters, extFilter) } else { - config.Filters = append(config.Filters, gzip.DefaultExtFilter()) + config.RequestFilters = append(config.RequestFilters, gzip.DefaultExtFilter()) + } + + // Response Filters + // If min_length is specified, use it. + if int64(lengthFilter) != 0 { + config.ResponseFilters = append(config.ResponseFilters, lengthFilter) } configs = append(configs, config) diff --git a/caddy/setup/gzip_test.go b/caddy/setup/gzip_test.go index 22d01d7a..36eeb0ae 100644 --- a/caddy/setup/gzip_test.go +++ b/caddy/setup/gzip_test.go @@ -73,6 +73,18 @@ func TestGzip(t *testing.T) { level 1 } `, false}, + {`gzip { not /file + ext * + level 1 + min_length ab + } + `, true}, + {`gzip { not /file + ext * + level 1 + min_length 1000 + } + `, false}, } for i, test := range tests { c := NewTestController(test.input) diff --git a/middleware/gzip/gzip.go b/middleware/gzip/gzip.go index e2d44775..b5866f68 100644 --- a/middleware/gzip/gzip.go +++ b/middleware/gzip/gzip.go @@ -23,8 +23,9 @@ type Gzip struct { // Config holds the configuration for Gzip middleware type Config struct { - Filters []Filter // Filters to use - Level int // Compression level + RequestFilters []RequestFilter + ResponseFilters []ResponseFilter + Level int // Compression level } // ServeHTTP serves a gzipped response if the client supports it. @@ -36,8 +37,8 @@ func (g Gzip) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) { outer: for _, c := range g.Configs { - // Check filters to determine if gzipping is permitted for this request - for _, filter := range c.Filters { + // Check request filters to determine if gzipping is permitted for this request + for _, filter := range c.RequestFilters { if !filter.ShouldCompress(r) { continue outer } @@ -56,8 +57,17 @@ outer: defer gzipWriter.Close() gz := gzipResponseWriter{Writer: gzipWriter, ResponseWriter: w} + var rw http.ResponseWriter + // if no response filter is used + if len(c.ResponseFilters) == 0 { + rw = gz + } else { + // wrap gzip writer with ResponseFilterWriter + rw = NewResponseFilterWriter(c.ResponseFilters, gz) + } + // Any response in forward middleware will now be compressed - status, err := g.Next.ServeHTTP(gz, r) + status, err := g.Next.ServeHTTP(rw, r) // If there was an error that remained unhandled, we need // to send something back before gzipWriter gets closed at diff --git a/middleware/gzip/gzip_test.go b/middleware/gzip/gzip_test.go index 3e7bed99..11ce6b20 100644 --- a/middleware/gzip/gzip_test.go +++ b/middleware/gzip/gzip_test.go @@ -21,7 +21,7 @@ func TestGzipHandler(t *testing.T) { extFilter.Exts.Add(e) } gz := Gzip{Configs: []Config{ - {Filters: []Filter{pathFilter, extFilter}}, + {RequestFilters: []RequestFilter{pathFilter, extFilter}}, }} w := httptest.NewRecorder() diff --git a/middleware/gzip/filter.go b/middleware/gzip/request_filter.go similarity index 91% rename from middleware/gzip/filter.go rename to middleware/gzip/request_filter.go index f4039da3..76a9ef83 100644 --- a/middleware/gzip/filter.go +++ b/middleware/gzip/request_filter.go @@ -7,8 +7,8 @@ import ( "github.com/mholt/caddy/middleware" ) -// Filter determines if a request should be gzipped. -type Filter interface { +// RequestFilter determines if a request should be gzipped. +type RequestFilter interface { // ShouldCompress tells if gzip compression // should be done on the request. ShouldCompress(*http.Request) bool @@ -26,7 +26,7 @@ func DefaultExtFilter() ExtFilter { return m } -// ExtFilter is Filter for file name extensions. +// ExtFilter is RequestFilter for file name extensions. type ExtFilter struct { // Exts is the file name extensions to accept Exts Set @@ -43,7 +43,7 @@ func (e ExtFilter) ShouldCompress(r *http.Request) bool { return e.Exts.Contains(ExtWildCard) || e.Exts.Contains(ext) } -// PathFilter is Filter for request path. +// PathFilter is RequestFilter for request path. type PathFilter struct { // IgnoredPaths is the paths to ignore IgnoredPaths Set diff --git a/middleware/gzip/filter_test.go b/middleware/gzip/request_filter_test.go similarity index 95% rename from middleware/gzip/filter_test.go rename to middleware/gzip/request_filter_test.go index f6537b9e..ce31d7fa 100644 --- a/middleware/gzip/filter_test.go +++ b/middleware/gzip/request_filter_test.go @@ -47,7 +47,7 @@ func TestSet(t *testing.T) { } func TestExtFilter(t *testing.T) { - var filter Filter = ExtFilter{make(Set)} + var filter RequestFilter = ExtFilter{make(Set)} for _, e := range []string{".txt", ".html", ".css", ".md"} { filter.(ExtFilter).Exts.Add(e) } @@ -86,7 +86,7 @@ func TestPathFilter(t *testing.T) { paths := []string{ "/a", "/b", "/c", "/de", } - var filter Filter = PathFilter{make(Set)} + var filter RequestFilter = PathFilter{make(Set)} for _, p := range paths { filter.(PathFilter).IgnoredPaths.Add(p) } diff --git a/middleware/gzip/response_filter.go b/middleware/gzip/response_filter.go new file mode 100644 index 00000000..8793dabe --- /dev/null +++ b/middleware/gzip/response_filter.go @@ -0,0 +1,62 @@ +package gzip + +import ( + "net/http" + "strconv" +) + +// ResponseFilter determines if the response should be gzipped. +type ResponseFilter interface { + ShouldCompress(http.ResponseWriter) bool +} + +// LengthFilter is ResponseFilter for minimum content length. +type LengthFilter int64 + +// ShouldCompress returns if content length is greater than or +// equals to minimum length. +func (l LengthFilter) ShouldCompress(w http.ResponseWriter) bool { + contentLength := (w.Header().Get("Content-Length")) + length, err := strconv.ParseInt(contentLength, 10, 64) + if err != nil || length == 0 { + return false + } + return l != 0 && int64(l) <= length +} + +// ResponseFilterWriter validates ResponseFilters. It writes +// gzip compressed data if ResponseFilters are satisfied or +// uncompressed data otherwise. +type ResponseFilterWriter struct { + filters []ResponseFilter + validated bool + shouldCompress bool + gzipResponseWriter +} + +// NewResponseFilterWriter creates and initializes a new ResponseFilterWriter. +func NewResponseFilterWriter(filters []ResponseFilter, gz gzipResponseWriter) *ResponseFilterWriter { + return &ResponseFilterWriter{filters: filters, gzipResponseWriter: gz} +} + +// Write wraps underlying Write method and compresses if filters +// are satisfied +func (r *ResponseFilterWriter) Write(b []byte) (int, error) { + // One time validation to determine if compression should + // be used or not. + if !r.validated { + r.shouldCompress = true + for _, filter := range r.filters { + if !filter.ShouldCompress(r) { + r.shouldCompress = false + break + } + } + r.validated = true + } + + if r.shouldCompress { + return r.gzipResponseWriter.Write(b) + } + return r.ResponseWriter.Write(b) +} diff --git a/middleware/gzip/response_filter_test.go b/middleware/gzip/response_filter_test.go new file mode 100644 index 00000000..1a5a1b4f --- /dev/null +++ b/middleware/gzip/response_filter_test.go @@ -0,0 +1,70 @@ +package gzip + +import ( + "compress/gzip" + "fmt" + "net/http/httptest" + "testing" +) + +func TestLengthFilter(t *testing.T) { + var filters []ResponseFilter = []ResponseFilter{ + LengthFilter(100), + LengthFilter(1000), + LengthFilter(0), + } + + var tests = []struct { + length int64 + shouldCompress [3]bool + }{ + {20, [3]bool{false, false, false}}, + {50, [3]bool{false, false, false}}, + {100, [3]bool{true, false, false}}, + {500, [3]bool{true, false, false}}, + {1000, [3]bool{true, true, false}}, + {1500, [3]bool{true, true, false}}, + } + + for i, ts := range tests { + for j, filter := range filters { + r := httptest.NewRecorder() + r.Header().Set("Content-Length", fmt.Sprint(ts.length)) + if filter.ShouldCompress(r) != ts.shouldCompress[j] { + t.Errorf("Test %v: Expected %v found %v", i, ts.shouldCompress[j], filter.ShouldCompress(r)) + } + } + } +} + +func TestResponseFilterWriter(t *testing.T) { + tests := []struct { + body string + shouldCompress bool + }{ + {"Hello\t\t\t\n", false}, + {"Hello the \t\t\t world is\n\n\n great", true}, + {"Hello \t\t\nfrom gzip", true}, + {"Hello gzip\n", false}, + } + filters := []ResponseFilter{ + LengthFilter(15), + } + for i, ts := range tests { + w := httptest.NewRecorder() + w.Header().Set("Content-Length", fmt.Sprint(len(ts.body))) + gz := gzipResponseWriter{gzip.NewWriter(w), w} + rw := NewResponseFilterWriter(filters, gz) + rw.Write([]byte(ts.body)) + resp := w.Body.String() + if !ts.shouldCompress { + if resp != ts.body { + t.Errorf("Test %v: No compression expected, found %v", i, resp) + } + } else { + if resp == ts.body { + t.Errorf("Test %v: Compression expected, found %v", i, resp) + } + } + } +} From 8631f33940a8c50b154da2756ce7d4954b925f0a Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Mon, 7 Dec 2015 23:27:57 +0100 Subject: [PATCH 13/44] remove minor ugly parenthesis --- middleware/gzip/response_filter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/middleware/gzip/response_filter.go b/middleware/gzip/response_filter.go index 8793dabe..fcfc8be5 100644 --- a/middleware/gzip/response_filter.go +++ b/middleware/gzip/response_filter.go @@ -16,7 +16,7 @@ type LengthFilter int64 // ShouldCompress returns if content length is greater than or // equals to minimum length. func (l LengthFilter) ShouldCompress(w http.ResponseWriter) bool { - contentLength := (w.Header().Get("Content-Length")) + contentLength := w.Header().Get("Content-Length") length, err := strconv.ParseInt(contentLength, 10, 64) if err != nil || length == 0 { return false From 23631cfaca8526192a7d18ce619a4fd8e46b4289 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Tue, 8 Dec 2015 12:01:24 +0100 Subject: [PATCH 14/44] Fix deleted Content-Length header bug. --- middleware/gzip/gzip.go | 17 ++++++++--- middleware/gzip/gzip_test.go | 2 ++ middleware/gzip/response_filter.go | 38 ++++++++++++++++--------- middleware/gzip/response_filter_test.go | 30 +++++++++++++++---- 4 files changed, 65 insertions(+), 22 deletions(-) diff --git a/middleware/gzip/gzip.go b/middleware/gzip/gzip.go index b5866f68..65256b2f 100644 --- a/middleware/gzip/gzip.go +++ b/middleware/gzip/gzip.go @@ -3,6 +3,7 @@ package gzip import ( + "bytes" "compress/gzip" "fmt" "io" @@ -47,9 +48,13 @@ outer: // Delete this header so gzipping is not repeated later in the chain r.Header.Del("Accept-Encoding") - w.Header().Set("Content-Encoding", "gzip") - w.Header().Set("Vary", "Accept-Encoding") - gzipWriter, err := newWriter(c, w) + // gzipWriter modifies underlying writer at init, + // use a buffer instead to leave ResponseWriter in + // original form. + var buf = &bytes.Buffer{} + defer buf.Reset() + + gzipWriter, err := newWriter(c, buf) if err != nil { // should not happen return http.StatusInternalServerError, err @@ -60,6 +65,8 @@ outer: var rw http.ResponseWriter // if no response filter is used if len(c.ResponseFilters) == 0 { + // replace buffer with ResponseWriter + gzipWriter.Reset(w) rw = gz } else { // wrap gzip writer with ResponseFilterWriter @@ -88,7 +95,7 @@ outer: // newWriter create a new Gzip Writer based on the compression level. // If the level is valid (i.e. between 1 and 9), it uses the level. // Otherwise, it uses default compression level. -func newWriter(c Config, w http.ResponseWriter) (*gzip.Writer, error) { +func newWriter(c Config, w io.Writer) (*gzip.Writer, error) { if c.Level >= gzip.BestSpeed && c.Level <= gzip.BestCompression { return gzip.NewWriterLevel(w, c.Level) } @@ -108,6 +115,8 @@ type gzipResponseWriter struct { // be wrong because it doesn't know it's being gzipped. func (w gzipResponseWriter) WriteHeader(code int) { w.Header().Del("Content-Length") + w.Header().Set("Content-Encoding", "gzip") + w.Header().Set("Vary", "Accept-Encoding") w.ResponseWriter.WriteHeader(code) } diff --git a/middleware/gzip/gzip_test.go b/middleware/gzip/gzip_test.go index 11ce6b20..c35c99c6 100644 --- a/middleware/gzip/gzip_test.go +++ b/middleware/gzip/gzip_test.go @@ -80,6 +80,8 @@ func TestGzipHandler(t *testing.T) { func nextFunc(shouldGzip bool) middleware.Handler { return middleware.HandlerFunc(func(w http.ResponseWriter, r *http.Request) (int, error) { + w.WriteHeader(200) + w.Write([]byte("test")) if shouldGzip { if r.Header.Get("Accept-Encoding") != "" { return 0, fmt.Errorf("Accept-Encoding header not expected") diff --git a/middleware/gzip/response_filter.go b/middleware/gzip/response_filter.go index fcfc8be5..034fd612 100644 --- a/middleware/gzip/response_filter.go +++ b/middleware/gzip/response_filter.go @@ -1,6 +1,7 @@ package gzip import ( + "compress/gzip" "net/http" "strconv" ) @@ -29,7 +30,6 @@ func (l LengthFilter) ShouldCompress(w http.ResponseWriter) bool { // uncompressed data otherwise. type ResponseFilterWriter struct { filters []ResponseFilter - validated bool shouldCompress bool gzipResponseWriter } @@ -40,21 +40,33 @@ func NewResponseFilterWriter(filters []ResponseFilter, gz gzipResponseWriter) *R } // Write wraps underlying Write method and compresses if filters -// are satisfied -func (r *ResponseFilterWriter) Write(b []byte) (int, error) { - // One time validation to determine if compression should - // be used or not. - if !r.validated { - r.shouldCompress = true - for _, filter := range r.filters { - if !filter.ShouldCompress(r) { - r.shouldCompress = false - break - } +// are satisfied. +func (r *ResponseFilterWriter) WriteHeader(code int) { + // Determine if compression should be used or not. + r.shouldCompress = true + for _, filter := range r.filters { + if !filter.ShouldCompress(r) { + r.shouldCompress = false + break } - r.validated = true } + if r.shouldCompress { + // replace buffer with ResponseWriter + if gzWriter, ok := r.gzipResponseWriter.Writer.(*gzip.Writer); ok { + gzWriter.Reset(r.ResponseWriter) + } + // use gzip WriteHeader to include and delete + // necessary headers + r.gzipResponseWriter.WriteHeader(code) + } else { + r.ResponseWriter.WriteHeader(code) + } +} + +// Write wraps underlying Write method and compresses if filters +// are satisfied +func (r *ResponseFilterWriter) Write(b []byte) (int, error) { if r.shouldCompress { return r.gzipResponseWriter.Write(b) } diff --git a/middleware/gzip/response_filter_test.go b/middleware/gzip/response_filter_test.go index 1a5a1b4f..cd7c7191 100644 --- a/middleware/gzip/response_filter_test.go +++ b/middleware/gzip/response_filter_test.go @@ -3,8 +3,11 @@ package gzip import ( "compress/gzip" "fmt" + "net/http" "net/http/httptest" "testing" + + "github.com/mholt/caddy/middleware" ) func TestLengthFilter(t *testing.T) { @@ -30,7 +33,8 @@ func TestLengthFilter(t *testing.T) { for j, filter := range filters { r := httptest.NewRecorder() r.Header().Set("Content-Length", fmt.Sprint(ts.length)) - if filter.ShouldCompress(r) != ts.shouldCompress[j] { + wWriter := NewResponseFilterWriter([]ResponseFilter{filter}, gzipResponseWriter{gzip.NewWriter(r), r}) + if filter.ShouldCompress(wWriter) != ts.shouldCompress[j] { t.Errorf("Test %v: Expected %v found %v", i, ts.shouldCompress[j], filter.ShouldCompress(r)) } } @@ -47,16 +51,32 @@ func TestResponseFilterWriter(t *testing.T) { {"Hello \t\t\nfrom gzip", true}, {"Hello gzip\n", false}, } + filters := []ResponseFilter{ LengthFilter(15), } + + server := Gzip{Configs: []Config{ + {ResponseFilters: filters}, + }} + for i, ts := range tests { + server.Next = middleware.HandlerFunc(func(w http.ResponseWriter, r *http.Request) (int, error) { + w.Header().Set("Content-Length", fmt.Sprint(len(ts.body))) + w.WriteHeader(200) + w.Write([]byte(ts.body)) + return 200, nil + }) + + r := urlRequest("/") + r.Header.Set("Accept-Encoding", "gzip") + w := httptest.NewRecorder() - w.Header().Set("Content-Length", fmt.Sprint(len(ts.body))) - gz := gzipResponseWriter{gzip.NewWriter(w), w} - rw := NewResponseFilterWriter(filters, gz) - rw.Write([]byte(ts.body)) + + server.ServeHTTP(w, r) + resp := w.Body.String() + if !ts.shouldCompress { if resp != ts.body { t.Errorf("Test %v: No compression expected, found %v", i, resp) From b6c4178f0ab7c49abd6e795708beddaf7b766137 Mon Sep 17 00:00:00 2001 From: Pavel Pavlenko Date: Wed, 9 Dec 2015 11:10:55 +0300 Subject: [PATCH 15/44] Remove ECDHE-RSA-3DES-EDE-CBC-SHA and RSA-3DES-EDE-CBC-SHA from the default TLS config --- caddy/setup/tls.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/caddy/setup/tls.go b/caddy/setup/tls.go index 4e5f7f9d..abec2d52 100644 --- a/caddy/setup/tls.go +++ b/caddy/setup/tls.go @@ -91,6 +91,9 @@ func SetDefaultTLSParams(c *server.Config) { // If no ciphers provided, use all that Caddy supports for the protocol if len(c.TLS.Ciphers) == 0 { c.TLS.Ciphers = supportedCiphers + + // Remove ECDHE-RSA-3DES-EDE-CBC-SHA and RSA-3DES-EDE-CBC-SHA from the default TLS config + c.TLS.Ciphers = c.TLS.Ciphers[:len(c.TLS.Ciphers)-2] } // Not a cipher suite, but still important for mitigating protocol downgrade attacks From e4ff77ed079a2a993b389097ff8dca259ea70404 Mon Sep 17 00:00:00 2001 From: Pavel Pavlenko Date: Wed, 9 Dec 2015 11:27:59 +0300 Subject: [PATCH 16/44] fix tls_test.go --- caddy/setup/tls_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/caddy/setup/tls_test.go b/caddy/setup/tls_test.go index fdea1e0c..62993701 100644 --- a/caddy/setup/tls_test.go +++ b/caddy/setup/tls_test.go @@ -42,15 +42,15 @@ func TestTLSParseBasic(t *testing.T) { tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, tls.TLS_RSA_WITH_AES_256_CBC_SHA, tls.TLS_RSA_WITH_AES_128_CBC_SHA, - tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, - tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, + //tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, + //tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, tls.TLS_FALLBACK_SCSV, } // Ensure count is correct (plus one for TLS_FALLBACK_SCSV) - if len(c.TLS.Ciphers) != len(supportedCiphers)+1 { + if len(c.TLS.Ciphers) != len(supportedCiphers)-1 { t.Errorf("Expected %v Ciphers (including TLS_FALLBACK_SCSV), got %v", - len(supportedCiphers)+1, len(c.TLS.Ciphers)) + len(supportedCiphers)-1, len(c.TLS.Ciphers)) } // Ensure ordering is correct From a44d59f1e553ed23a489da63c64f0932c7ccc295 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Wed, 9 Dec 2015 18:44:25 +0100 Subject: [PATCH 17/44] Use ioutil.Discard instead for unneeded bytes. --- middleware/gzip/gzip.go | 11 ++++------- middleware/gzip/response_filter.go | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/middleware/gzip/gzip.go b/middleware/gzip/gzip.go index 65256b2f..147d739f 100644 --- a/middleware/gzip/gzip.go +++ b/middleware/gzip/gzip.go @@ -3,10 +3,10 @@ package gzip import ( - "bytes" "compress/gzip" "fmt" "io" + "io/ioutil" "net/http" "strings" @@ -49,12 +49,9 @@ outer: r.Header.Del("Accept-Encoding") // gzipWriter modifies underlying writer at init, - // use a buffer instead to leave ResponseWriter in + // use a discard writer instead to leave ResponseWriter in // original form. - var buf = &bytes.Buffer{} - defer buf.Reset() - - gzipWriter, err := newWriter(c, buf) + gzipWriter, err := newWriter(c, ioutil.Discard) if err != nil { // should not happen return http.StatusInternalServerError, err @@ -65,7 +62,7 @@ outer: var rw http.ResponseWriter // if no response filter is used if len(c.ResponseFilters) == 0 { - // replace buffer with ResponseWriter + // replace discard writer with ResponseWriter gzipWriter.Reset(w) rw = gz } else { diff --git a/middleware/gzip/response_filter.go b/middleware/gzip/response_filter.go index 034fd612..c599b3e1 100644 --- a/middleware/gzip/response_filter.go +++ b/middleware/gzip/response_filter.go @@ -52,7 +52,7 @@ func (r *ResponseFilterWriter) WriteHeader(code int) { } if r.shouldCompress { - // replace buffer with ResponseWriter + // replace discard writer with ResponseWriter if gzWriter, ok := r.gzipResponseWriter.Writer.(*gzip.Writer); ok { gzWriter.Reset(r.ResponseWriter) } From 59dbea768c75a7ce07453010fcf6c392738771b8 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Wed, 9 Dec 2015 19:22:20 +0100 Subject: [PATCH 18/44] Fix race condition on AppVeyor. Increase timeout a bit. --- middleware/markdown/watcher_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/middleware/markdown/watcher_test.go b/middleware/markdown/watcher_test.go index d09d07c6..2f7ca4cb 100644 --- a/middleware/markdown/watcher_test.go +++ b/middleware/markdown/watcher_test.go @@ -18,7 +18,7 @@ func TestWatcher(t *testing.T) { out += fmt.Sprint(i) }) // wait little more because of concurrency - time.Sleep(interval * 9) + time.Sleep(interval * 12) stopChan <- struct{}{} if !strings.HasPrefix(out, expected) { t.Fatalf("Expected to have prefix %v, found %v", expected, out) @@ -32,7 +32,7 @@ func TestWatcher(t *testing.T) { out += fmt.Sprint(i) mu.Unlock() }) - time.Sleep(interval * 10) + time.Sleep(interval * 15) mu.Lock() res := out mu.Unlock() From 2e295b51b3344ead5a980827a8f26d9ec9c9b1de Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Wed, 9 Dec 2015 20:18:18 +0100 Subject: [PATCH 19/44] Use channel instead for a synchronous interval. --- middleware/markdown/watcher_test.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/middleware/markdown/watcher_test.go b/middleware/markdown/watcher_test.go index 2f7ca4cb..8a89e007 100644 --- a/middleware/markdown/watcher_test.go +++ b/middleware/markdown/watcher_test.go @@ -13,14 +13,14 @@ func TestWatcher(t *testing.T) { interval := time.Millisecond * 100 i := 0 out := "" + syncChan := make(chan struct{}) stopChan := TickerFunc(interval, func() { i++ out += fmt.Sprint(i) + syncChan <- struct{}{} }) - // wait little more because of concurrency - time.Sleep(interval * 12) - stopChan <- struct{}{} - if !strings.HasPrefix(out, expected) { + sleepInSync(8, syncChan, stopChan) + if out != expected { t.Fatalf("Expected to have prefix %v, found %v", expected, out) } out = "" @@ -31,8 +31,9 @@ func TestWatcher(t *testing.T) { mu.Lock() out += fmt.Sprint(i) mu.Unlock() + syncChan <- struct{}{} }) - time.Sleep(interval * 15) + sleepInSync(9, syncChan, stopChan) mu.Lock() res := out mu.Unlock() @@ -40,3 +41,10 @@ func TestWatcher(t *testing.T) { t.Fatalf("expected (%v) must be a proper prefix of out(%v).", expected, out) } } + +func sleepInSync(times int, syncChan chan struct{}, stopChan chan struct{}) { + for i := 0; i < times; i++ { + <-syncChan + } + stopChan <- struct{}{} +} From afbda595f6c031ebb09b013858895167b79803c9 Mon Sep 17 00:00:00 2001 From: Radim Marek Date: Fri, 11 Dec 2015 21:47:38 +0100 Subject: [PATCH 20/44] import glob tests --- caddy/parse/import_glob0.txt | 6 ++++++ caddy/parse/import_glob1.txt | 4 ++++ caddy/parse/import_glob2.txt | 3 +++ caddy/parse/parsing_test.go | 7 +++++++ 4 files changed, 20 insertions(+) create mode 100644 caddy/parse/import_glob0.txt create mode 100644 caddy/parse/import_glob1.txt create mode 100644 caddy/parse/import_glob2.txt diff --git a/caddy/parse/import_glob0.txt b/caddy/parse/import_glob0.txt new file mode 100644 index 00000000..e610b5e7 --- /dev/null +++ b/caddy/parse/import_glob0.txt @@ -0,0 +1,6 @@ +glob0.host0 { + dir2 arg1 +} + +glob0.host1 { +} diff --git a/caddy/parse/import_glob1.txt b/caddy/parse/import_glob1.txt new file mode 100644 index 00000000..111eb044 --- /dev/null +++ b/caddy/parse/import_glob1.txt @@ -0,0 +1,4 @@ +glob1.host0 { + dir1 + dir2 arg1 +} diff --git a/caddy/parse/import_glob2.txt b/caddy/parse/import_glob2.txt new file mode 100644 index 00000000..c09f784e --- /dev/null +++ b/caddy/parse/import_glob2.txt @@ -0,0 +1,3 @@ +glob2.host0 { + dir2 arg1 +} diff --git a/caddy/parse/parsing_test.go b/caddy/parse/parsing_test.go index 97c86808..bda6b29b 100644 --- a/caddy/parse/parsing_test.go +++ b/caddy/parse/parsing_test.go @@ -329,6 +329,13 @@ func TestParseAll(t *testing.T) { []address{{"host1.com", "http"}, {"host2.com", "http"}}, []address{{"host3.com", "https"}, {"host4.com", "https"}}, }}, + + {`import import_glob*.txt`, false, [][]address{ + []address{{"glob0.host0", ""}}, + []address{{"glob0.host1", ""}}, + []address{{"glob1.host0", ""}}, + []address{{"glob2.host0", ""}}, + }}, } { p := testParser(test.input) blocks, err := p.parseAll() From eb48885d4d24305e183223dbe3e9d947125d3274 Mon Sep 17 00:00:00 2001 From: Radim Marek Date: Fri, 11 Dec 2015 22:02:31 +0100 Subject: [PATCH 21/44] Updated comments --- caddy/parse/parsing.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/caddy/parse/parsing.go b/caddy/parse/parsing.go index e011fd8c..db4b7e42 100644 --- a/caddy/parse/parsing.go +++ b/caddy/parse/parsing.go @@ -176,10 +176,11 @@ func (p *parser) directives() error { } // doImport swaps out the import directive and its argument -// (a total of 2 tokens) with the tokens in the file specified. -// When the function returns, the cursor is on the token before -// where the import directive was. In other words, call Next() -// to access the first token that was imported. +// (a total of 2 tokens) with the tokens in the specified file +// or globbing pattern. When the function returns, the cursor +// is on the token before where the import directive was. In +// other words, call Next() to access the first token that was +// imported. func (p *parser) doImport() error { if !p.NextArg() { return p.ArgErr() @@ -218,6 +219,8 @@ func (p *parser) doImport() error { return nil } +// doSingleImport lexes the individual files matching the +// globbing pattern from of the import directive. func (p *parser) doSingleImport(importFile string) error { file, err := os.Open(importFile) if err != nil { From b7fd1f4e9ea044566e246b0768f2254238400d3e Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Sat, 12 Dec 2015 14:04:48 +0100 Subject: [PATCH 22/44] FastCGI: separate standard and error output responses. --- middleware/fastcgi/fastcgi.go | 15 ++++++++++++++- middleware/fastcgi/fcgiclient.go | 21 +++++++++++++++++---- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/middleware/fastcgi/fastcgi.go b/middleware/fastcgi/fastcgi.go index 77fb668f..b7a86b66 100755 --- a/middleware/fastcgi/fastcgi.go +++ b/middleware/fastcgi/fastcgi.go @@ -115,7 +115,12 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) return http.StatusBadGateway, err } - return 0, nil + // FastCGI stderr outputs + if fcgi.stderr.Len() != 0 { + err = LogError(fcgi.stderr.String()) + } + + return 0, err } } @@ -281,3 +286,11 @@ var ( // ErrIndexMissingSplit describes an index configuration error. ErrIndexMissingSplit = errors.New("configured index file(s) must include split value") ) + +// LogError is a non fatal error that allows requests to go through. +type LogError string + +// Error satisfies error interface. +func (l LogError) Error() string { + return string(l) +} diff --git a/middleware/fastcgi/fcgiclient.go b/middleware/fastcgi/fcgiclient.go index b8b03fe7..97dc6153 100644 --- a/middleware/fastcgi/fcgiclient.go +++ b/middleware/fastcgi/fcgiclient.go @@ -164,6 +164,7 @@ type FCGIClient struct { rwc io.ReadWriteCloser h header buf bytes.Buffer + stderr bytes.Buffer keepAlive bool reqID uint16 } @@ -346,10 +347,22 @@ func (w *streamReader) Read(p []byte) (n int, err error) { if len(p) > 0 { if len(w.buf) == 0 { - rec := &record{} - w.buf, err = rec.read(w.c.rwc) - if err != nil { - return + + // filter outputs for error log + for { + rec := &record{} + var buf []byte + buf, err = rec.read(w.c.rwc) + if err != nil { + return + } + // standard error output + if rec.h.Type == Stderr { + w.c.stderr.Write(buf) + continue + } + w.buf = buf + break } } From 3966936bd6f01462fb8b41198bf36a83e17ad6e7 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Sun, 13 Dec 2015 12:58:21 +0100 Subject: [PATCH 23/44] Remove trailing new line for cleaner log output. Return correct status code. --- middleware/fastcgi/fastcgi.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/middleware/fastcgi/fastcgi.go b/middleware/fastcgi/fastcgi.go index b7a86b66..21404c2e 100755 --- a/middleware/fastcgi/fastcgi.go +++ b/middleware/fastcgi/fastcgi.go @@ -117,10 +117,11 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) // FastCGI stderr outputs if fcgi.stderr.Len() != 0 { - err = LogError(fcgi.stderr.String()) + // Remove trailing newline, error logger already does this. + err = LogError(strings.TrimSuffix(fcgi.stderr.String(), "\n")) } - return 0, err + return resp.StatusCode, err } } From 34d3cd7c927e113c7ec65735c4694fb21d2d51ae Mon Sep 17 00:00:00 2001 From: Craig Peterson Date: Tue, 15 Dec 2015 08:56:44 -0700 Subject: [PATCH 24/44] Gzip: Append to Vary header instead of replacing. --- middleware/gzip/gzip.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/middleware/gzip/gzip.go b/middleware/gzip/gzip.go index 147d739f..39a92266 100644 --- a/middleware/gzip/gzip.go +++ b/middleware/gzip/gzip.go @@ -113,7 +113,7 @@ type gzipResponseWriter struct { func (w gzipResponseWriter) WriteHeader(code int) { w.Header().Del("Content-Length") w.Header().Set("Content-Encoding", "gzip") - w.Header().Set("Vary", "Accept-Encoding") + w.Header().Add("Vary", "Accept-Encoding") w.ResponseWriter.WriteHeader(code) } From 5eadea6615b4667a003cc977b8f1c3d8ef1ce47c Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 17 Dec 2015 08:37:16 -0700 Subject: [PATCH 25/44] Slack -> Gitter --- CONTRIBUTING.md | 22 ++++++++++++---------- README.md | 8 +++++--- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1b4d3f3f..b05a6e0f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,8 +1,9 @@ ## Contributing to Caddy -**[Join us on Slack](https://gophers.slack.com/messages/caddy/)** to chat with -other Caddy developers! ([Request an invite](http://bit.ly/go-slack-signup), -then join the #caddy channel.) +**[Join our dev chat on Gitter](https://gitter.im/mholt/caddy)** to chat with +other Caddy developers! (Dev chat only; try our +[support room](https://gitter.im/caddyserver/support) for help or +[general](https://gitter.im/caddyserver/general) for anything else.) This project gladly accepts contributions and we encourage interested users to get involved! @@ -11,24 +12,25 @@ get involved! #### For small tweaks, bug fixes, and tests Submit [pull requests](https://github.com/mholt/caddy/pulls) at any time. -Thank you for helping out in simple ways! Bug fixes should be under test to -assert correct behavior. +Bug fixes should be under test to assert correct behavior. Thank you for +helping out in simple ways! #### Ideas, questions, bug reports -You should totally [open an issue](https://github.com/mholt/caddy/issues) with -your ideas, questions, and bug reports, if one does not already exist for it. -Bug reports should state expected behavior and contain clear instructions for -reproducing the problem. +Feek free to [open an issue](https://github.com/mholt/caddy/issues) with your +ideas, questions, and bug reports, if one does not already exist for it. Bug +reports should state expected behavior and contain clear instructions for +isolating and reproducing the problem. See [How to Report Bugs Effectively](http://www.chiark.greenend.org.uk/~sgtatham/bugs.html). + #### New features Before submitting a pull request, please open an issue first to discuss it and claim it. This prevents overlapping efforts and keeps the project in-line with its goals. If you prefer to discuss the feature privately, you can reach other -developers on Slack or you may email me directly. (My email address is below.) +developers on Gitter or you may email me directly. (My email address is below.) And don't forget to write tests for new features! diff --git a/README.md b/README.md index cb81c175..6aa9510a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Caddy](https://caddyserver.com/resources/images/caddy-boxed.png)](https://caddyserver.com) +[![Dev Chat](https://img.shields.io/badge/dev%20chat-gitter-ff69b4.svg?style=flat-square&label=dev+chat&color=ff69b4)](https://gitter.im/mholt/caddy) [![Documentation](https://img.shields.io/badge/godoc-reference-blue.svg?style=flat-square)](https://godoc.org/github.com/mholt/caddy) [![Linux Build Status](https://img.shields.io/travis/mholt/caddy.svg?style=flat-square&label=linux+build)](https://travis-ci.org/mholt/caddy) [![Windows Build Status](https://img.shields.io/appveyor/ci/mholt/caddy.svg?style=flat-square&label=windows+build)](https://ci.appveyor.com/project/mholt/caddy) @@ -139,9 +140,10 @@ packages that each Caddy package imports. ## Contributing -**[Join us on Slack](https://gophers.slack.com/messages/caddy/)** to chat with -other Caddy developers! ([Request an invite](http://bit.ly/go-slack-signup), -then join the #caddy channel.) +**[Join our dev chat on Gitter](https://gitter.im/mholt/caddy)** to chat with +other Caddy developers! (Dev chat only; try our +[support room](https://gitter.im/caddyserver/support) for help or +[general](https://gitter.im/caddyserver/general) for anything else.) This project would not be what it is without your help. Please see the [contributing guidelines](https://github.com/mholt/caddy/blob/master/CONTRIBUTING.md) From 8f23c430ae3a3f9101ea823a3b6a00b985dded3c Mon Sep 17 00:00:00 2001 From: jungle-boogie Date: Thu, 17 Dec 2015 13:12:22 -0800 Subject: [PATCH 26/44] Feek --> Feel unless you meant Freak --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b05a6e0f..44eb8638 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,7 +18,7 @@ helping out in simple ways! #### Ideas, questions, bug reports -Feek free to [open an issue](https://github.com/mholt/caddy/issues) with your +Feel free to [open an issue](https://github.com/mholt/caddy/issues) with your ideas, questions, and bug reports, if one does not already exist for it. Bug reports should state expected behavior and contain clear instructions for isolating and reproducing the problem. From f04ff063ed62b8ce0b2c3cafdf6bcb90fffd4a4b Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Fri, 18 Dec 2015 20:58:23 +0100 Subject: [PATCH 27/44] Gzip: Fix missing gzip encoding headers. --- middleware/gzip/gzip.go | 11 ++++++++--- middleware/gzip/gzip_test.go | 4 ++-- middleware/gzip/response_filter.go | 4 ++-- middleware/gzip/response_filter_test.go | 2 +- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/middleware/gzip/gzip.go b/middleware/gzip/gzip.go index 147d739f..99903ff0 100644 --- a/middleware/gzip/gzip.go +++ b/middleware/gzip/gzip.go @@ -57,7 +57,7 @@ outer: return http.StatusInternalServerError, err } defer gzipWriter.Close() - gz := gzipResponseWriter{Writer: gzipWriter, ResponseWriter: w} + gz := &gzipResponseWriter{Writer: gzipWriter, ResponseWriter: w} var rw http.ResponseWriter // if no response filter is used @@ -104,21 +104,26 @@ func newWriter(c Config, w io.Writer) (*gzip.Writer, error) { type gzipResponseWriter struct { io.Writer http.ResponseWriter + statusCodeWritten bool } // WriteHeader wraps the underlying WriteHeader method to prevent // problems with conflicting headers from proxied backends. For // example, a backend system that calculates Content-Length would // be wrong because it doesn't know it's being gzipped. -func (w gzipResponseWriter) WriteHeader(code int) { +func (w *gzipResponseWriter) WriteHeader(code int) { w.Header().Del("Content-Length") w.Header().Set("Content-Encoding", "gzip") w.Header().Set("Vary", "Accept-Encoding") w.ResponseWriter.WriteHeader(code) + w.statusCodeWritten = true } // Write wraps the underlying Write method to do compression. -func (w gzipResponseWriter) Write(b []byte) (int, error) { +func (w *gzipResponseWriter) Write(b []byte) (int, error) { + if !w.statusCodeWritten { + w.WriteHeader(http.StatusOK) + } if w.Header().Get("Content-Type") == "" { w.Header().Set("Content-Type", http.DetectContentType(b)) } diff --git a/middleware/gzip/gzip_test.go b/middleware/gzip/gzip_test.go index c35c99c6..b9bafc6d 100644 --- a/middleware/gzip/gzip_test.go +++ b/middleware/gzip/gzip_test.go @@ -92,7 +92,7 @@ func nextFunc(shouldGzip bool) middleware.Handler { if w.Header().Get("Vary") != "Accept-Encoding" { return 0, fmt.Errorf("Vary must be Accept-Encoding, found %v", r.Header.Get("Vary")) } - if _, ok := w.(gzipResponseWriter); !ok { + if _, ok := w.(*gzipResponseWriter); !ok { return 0, fmt.Errorf("ResponseWriter should be gzipResponseWriter, found %T", w) } return 0, nil @@ -103,7 +103,7 @@ func nextFunc(shouldGzip bool) middleware.Handler { if w.Header().Get("Content-Encoding") == "gzip" { return 0, fmt.Errorf("Content-Encoding must not be gzip, found gzip") } - if _, ok := w.(gzipResponseWriter); ok { + if _, ok := w.(*gzipResponseWriter); ok { return 0, fmt.Errorf("ResponseWriter should not be gzipResponseWriter") } return 0, nil diff --git a/middleware/gzip/response_filter.go b/middleware/gzip/response_filter.go index c599b3e1..87a34e60 100644 --- a/middleware/gzip/response_filter.go +++ b/middleware/gzip/response_filter.go @@ -31,11 +31,11 @@ func (l LengthFilter) ShouldCompress(w http.ResponseWriter) bool { type ResponseFilterWriter struct { filters []ResponseFilter shouldCompress bool - gzipResponseWriter + *gzipResponseWriter } // NewResponseFilterWriter creates and initializes a new ResponseFilterWriter. -func NewResponseFilterWriter(filters []ResponseFilter, gz gzipResponseWriter) *ResponseFilterWriter { +func NewResponseFilterWriter(filters []ResponseFilter, gz *gzipResponseWriter) *ResponseFilterWriter { return &ResponseFilterWriter{filters: filters, gzipResponseWriter: gz} } diff --git a/middleware/gzip/response_filter_test.go b/middleware/gzip/response_filter_test.go index cd7c7191..73867e7f 100644 --- a/middleware/gzip/response_filter_test.go +++ b/middleware/gzip/response_filter_test.go @@ -33,7 +33,7 @@ func TestLengthFilter(t *testing.T) { for j, filter := range filters { r := httptest.NewRecorder() r.Header().Set("Content-Length", fmt.Sprint(ts.length)) - wWriter := NewResponseFilterWriter([]ResponseFilter{filter}, gzipResponseWriter{gzip.NewWriter(r), r}) + wWriter := NewResponseFilterWriter([]ResponseFilter{filter}, &gzipResponseWriter{gzip.NewWriter(r), r, false}) if filter.ShouldCompress(wWriter) != ts.shouldCompress[j] { t.Errorf("Test %v: Expected %v found %v", i, ts.shouldCompress[j], filter.ShouldCompress(r)) } From a946d65fe6ba95fe02407b6f4a7c4478c46bd50b Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Fri, 18 Dec 2015 21:25:06 +0100 Subject: [PATCH 28/44] Oops. Tests. --- middleware/gzip/gzip_test.go | 1 - middleware/gzip/response_filter.go | 11 ++++++++--- middleware/gzip/response_filter_test.go | 1 - 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/middleware/gzip/gzip_test.go b/middleware/gzip/gzip_test.go index b9bafc6d..0e4df338 100644 --- a/middleware/gzip/gzip_test.go +++ b/middleware/gzip/gzip_test.go @@ -80,7 +80,6 @@ func TestGzipHandler(t *testing.T) { func nextFunc(shouldGzip bool) middleware.Handler { return middleware.HandlerFunc(func(w http.ResponseWriter, r *http.Request) (int, error) { - w.WriteHeader(200) w.Write([]byte("test")) if shouldGzip { if r.Header.Get("Accept-Encoding") != "" { diff --git a/middleware/gzip/response_filter.go b/middleware/gzip/response_filter.go index 87a34e60..b561649e 100644 --- a/middleware/gzip/response_filter.go +++ b/middleware/gzip/response_filter.go @@ -29,8 +29,9 @@ func (l LengthFilter) ShouldCompress(w http.ResponseWriter) bool { // gzip compressed data if ResponseFilters are satisfied or // uncompressed data otherwise. type ResponseFilterWriter struct { - filters []ResponseFilter - shouldCompress bool + filters []ResponseFilter + shouldCompress bool + statusCodeWritten bool *gzipResponseWriter } @@ -39,7 +40,7 @@ func NewResponseFilterWriter(filters []ResponseFilter, gz *gzipResponseWriter) * return &ResponseFilterWriter{filters: filters, gzipResponseWriter: gz} } -// Write wraps underlying Write method and compresses if filters +// Write wraps underlying WriteHeader method and compresses if filters // are satisfied. func (r *ResponseFilterWriter) WriteHeader(code int) { // Determine if compression should be used or not. @@ -62,11 +63,15 @@ func (r *ResponseFilterWriter) WriteHeader(code int) { } else { r.ResponseWriter.WriteHeader(code) } + r.statusCodeWritten = true } // Write wraps underlying Write method and compresses if filters // are satisfied func (r *ResponseFilterWriter) Write(b []byte) (int, error) { + if !r.statusCodeWritten { + r.WriteHeader(http.StatusOK) + } if r.shouldCompress { return r.gzipResponseWriter.Write(b) } diff --git a/middleware/gzip/response_filter_test.go b/middleware/gzip/response_filter_test.go index 73867e7f..75f72692 100644 --- a/middleware/gzip/response_filter_test.go +++ b/middleware/gzip/response_filter_test.go @@ -63,7 +63,6 @@ func TestResponseFilterWriter(t *testing.T) { for i, ts := range tests { server.Next = middleware.HandlerFunc(func(w http.ResponseWriter, r *http.Request) (int, error) { w.Header().Set("Content-Length", fmt.Sprint(len(ts.body))) - w.WriteHeader(200) w.Write([]byte(ts.body)) return 200, nil }) From 1e27b5be8907021b44b307f216f6b86df5db1adc Mon Sep 17 00:00:00 2001 From: Pavel Pavlenko Date: Sat, 19 Dec 2015 14:30:25 +0300 Subject: [PATCH 29/44] Remove ECDHE-RSA-3DES-EDE-CBC-SHA and RSA-3DES-EDE-CBC-SHA from the default TLS config --- caddy/setup/tls.go | 17 +++++++++++++---- caddy/setup/tls_test.go | 6 ++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/caddy/setup/tls.go b/caddy/setup/tls.go index abec2d52..79954de4 100644 --- a/caddy/setup/tls.go +++ b/caddy/setup/tls.go @@ -90,10 +90,7 @@ func TLS(c *Controller) (middleware.Middleware, error) { func SetDefaultTLSParams(c *server.Config) { // If no ciphers provided, use all that Caddy supports for the protocol if len(c.TLS.Ciphers) == 0 { - c.TLS.Ciphers = supportedCiphers - - // Remove ECDHE-RSA-3DES-EDE-CBC-SHA and RSA-3DES-EDE-CBC-SHA from the default TLS config - c.TLS.Ciphers = c.TLS.Ciphers[:len(c.TLS.Ciphers)-2] + c.TLS.Ciphers = defaultCiphers } // Not a cipher suite, but still important for mitigating protocol downgrade attacks @@ -162,3 +159,15 @@ var supportedCiphers = []uint16{ tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, } + +// List of all the ciphers we want to use by default +var defaultCiphers = []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + tls.TLS_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_RSA_WITH_AES_128_CBC_SHA, +} diff --git a/caddy/setup/tls_test.go b/caddy/setup/tls_test.go index 62993701..8e2ecece 100644 --- a/caddy/setup/tls_test.go +++ b/caddy/setup/tls_test.go @@ -42,15 +42,13 @@ func TestTLSParseBasic(t *testing.T) { tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, tls.TLS_RSA_WITH_AES_256_CBC_SHA, tls.TLS_RSA_WITH_AES_128_CBC_SHA, - //tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA, - //tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA, tls.TLS_FALLBACK_SCSV, } // Ensure count is correct (plus one for TLS_FALLBACK_SCSV) - if len(c.TLS.Ciphers) != len(supportedCiphers)-1 { + if len(c.TLS.Ciphers) != len(defaultCiphers) { t.Errorf("Expected %v Ciphers (including TLS_FALLBACK_SCSV), got %v", - len(supportedCiphers)-1, len(c.TLS.Ciphers)) + len(defaultCiphers), len(c.TLS.Ciphers)) } // Ensure ordering is correct From 3dd4c0eb6a34e6fe0929296bd908fa7b31662406 Mon Sep 17 00:00:00 2001 From: Pavel Pavlenko Date: Sat, 19 Dec 2015 14:37:38 +0300 Subject: [PATCH 30/44] Fix TestTLSParseBasic --- caddy/setup/tls_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/caddy/setup/tls_test.go b/caddy/setup/tls_test.go index 8e2ecece..e2d2e015 100644 --- a/caddy/setup/tls_test.go +++ b/caddy/setup/tls_test.go @@ -46,9 +46,9 @@ func TestTLSParseBasic(t *testing.T) { } // Ensure count is correct (plus one for TLS_FALLBACK_SCSV) - if len(c.TLS.Ciphers) != len(defaultCiphers) { + if len(c.TLS.Ciphers) != len(expectedCiphers) { t.Errorf("Expected %v Ciphers (including TLS_FALLBACK_SCSV), got %v", - len(defaultCiphers), len(c.TLS.Ciphers)) + len(expectedCiphers), len(c.TLS.Ciphers)) } // Ensure ordering is correct From f639d3cd68bc8bf06fe7953b3278d206a2f17424 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Mon, 21 Dec 2015 11:57:20 +0100 Subject: [PATCH 31/44] FastCGI: Close client connections when done. --- middleware/fastcgi/fcgiclient.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/middleware/fastcgi/fcgiclient.go b/middleware/fastcgi/fcgiclient.go index 97dc6153..511a5219 100644 --- a/middleware/fastcgi/fcgiclient.go +++ b/middleware/fastcgi/fcgiclient.go @@ -400,6 +400,15 @@ func (c *FCGIClient) Do(p map[string]string, req io.Reader) (r io.Reader, err er return } +// clientCloser is a io.ReadCloser. It wraps a io.Reader with a Closer +// that closes FCGIClient connection. +type clientCloser struct { + *FCGIClient + io.Reader +} + +func (f clientCloser) Close() error { return f.rwc.Close() } + // Request returns a HTTP Response with Header and Body // from fcgi responder func (c *FCGIClient) Request(p map[string]string, req io.Reader) (resp *http.Response, err error) { @@ -439,9 +448,9 @@ func (c *FCGIClient) Request(p map[string]string, req io.Reader) (resp *http.Res resp.ContentLength, _ = strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64) if chunked(resp.TransferEncoding) { - resp.Body = ioutil.NopCloser(httputil.NewChunkedReader(rb)) + resp.Body = clientCloser{c, httputil.NewChunkedReader(rb)} } else { - resp.Body = ioutil.NopCloser(rb) + resp.Body = clientCloser{c, ioutil.NopCloser(rb)} } return } From 4d867e848bbc6d4ff65f3a4b840ad2349579adcb Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Tue, 22 Dec 2015 13:32:27 +0100 Subject: [PATCH 32/44] Markdown: Fix panic on sitegen for request dependent template values. --- middleware/markdown/generator.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/middleware/markdown/generator.go b/middleware/markdown/generator.go index d3485e91..653f457f 100644 --- a/middleware/markdown/generator.go +++ b/middleware/markdown/generator.go @@ -5,6 +5,8 @@ import ( "encoding/hex" "fmt" "io/ioutil" + "net/http" + "net/url" "os" "path/filepath" "strings" @@ -104,7 +106,7 @@ func generateStaticHTML(md Markdown, cfg *Config) error { reqPath = "/" + reqPath // Generate the static file - ctx := middleware.Context{Root: md.FileSys} + ctx := middleware.Context{Root: md.FileSys, Req: new(http.Request), URL: new(url.URL)} _, err = md.Process(cfg, reqPath, body, ctx) if err != nil { return err From 9e163a655d379eb6d19120a5de3dfd034a0f7eec Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Tue, 22 Dec 2015 14:43:48 +0100 Subject: [PATCH 33/44] Use proper struct constructors instead. --- middleware/markdown/generator.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/middleware/markdown/generator.go b/middleware/markdown/generator.go index 653f457f..d218f22b 100644 --- a/middleware/markdown/generator.go +++ b/middleware/markdown/generator.go @@ -105,8 +105,12 @@ func generateStaticHTML(md Markdown, cfg *Config) error { reqPath = filepath.ToSlash(reqPath) reqPath = "/" + reqPath + // Create empty requests and url to cater for template values. + req, _ := http.NewRequest("", "/", nil) + urlVar, _ := url.Parse("/") + // Generate the static file - ctx := middleware.Context{Root: md.FileSys, Req: new(http.Request), URL: new(url.URL)} + ctx := middleware.Context{Root: md.FileSys, Req: req, URL: urlVar} _, err = md.Process(cfg, reqPath, body, ctx) if err != nil { return err From 98d8c0f81b6fbac4a818368861b44c8ee42d22a3 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Tue, 22 Dec 2015 23:19:22 +0100 Subject: [PATCH 34/44] Added new rewrite features. --- caddy/setup/rewrite.go | 22 +++++- caddy/setup/rewrite_test.go | 20 +++--- middleware/rewrite/condition.go | 110 +++++++++++++++++++++++++++++ middleware/rewrite/rewrite.go | 78 +++++++++----------- middleware/rewrite/rewrite_test.go | 5 +- middleware/rewrite/to.go | 68 ++++++++++++++++++ 6 files changed, 243 insertions(+), 60 deletions(-) create mode 100644 middleware/rewrite/condition.go create mode 100644 middleware/rewrite/to.go diff --git a/caddy/setup/rewrite.go b/caddy/setup/rewrite.go index b510a237..c70a2f94 100644 --- a/caddy/setup/rewrite.go +++ b/caddy/setup/rewrite.go @@ -1,6 +1,8 @@ package setup import ( + "net/http" + "github.com/mholt/caddy/middleware" "github.com/mholt/caddy/middleware/rewrite" ) @@ -13,7 +15,11 @@ func Rewrite(c *Controller) (middleware.Middleware, error) { } return func(next middleware.Handler) middleware.Handler { - return rewrite.Rewrite{Next: next, Rules: rewrites} + return rewrite.Rewrite{ + Next: next, + FileSys: http.Dir(c.Root), + Rules: rewrites, + } }, nil } @@ -30,6 +36,8 @@ func rewriteParse(c *Controller) ([]rewrite.Rule, error) { args := c.RemainingArgs() + var ifs []rewrite.If + switch len(args) { case 2: rule = rewrite.NewSimpleRule(args[0], args[1]) @@ -56,6 +64,16 @@ func rewriteParse(c *Controller) ([]rewrite.Rule, error) { return nil, c.ArgErr() } ext = args1 + case "if": + args1 := c.RemainingArgs() + if len(args1) != 3 { + return nil, c.ArgErr() + } + ifCond, err := rewrite.NewIf(args1[0], args1[1], args1[2]) + if err != nil { + return nil, err + } + ifs = append(ifs, ifCond) default: return nil, c.ArgErr() } @@ -64,7 +82,7 @@ func rewriteParse(c *Controller) ([]rewrite.Rule, error) { if pattern == "" || to == "" { return nil, c.ArgErr() } - if rule, err = rewrite.NewRegexpRule(base, pattern, to, ext); err != nil { + if rule, err = rewrite.NewComplexRule(base, pattern, to, ext, ifs); err != nil { return nil, err } regexpRules = append(regexpRules, rule) diff --git a/caddy/setup/rewrite_test.go b/caddy/setup/rewrite_test.go index f5426678..ddef7cd4 100644 --- a/caddy/setup/rewrite_test.go +++ b/caddy/setup/rewrite_test.go @@ -98,14 +98,14 @@ func TestRewriteParse(t *testing.T) { r .* to /to }`, false, []rewrite.Rule{ - &rewrite.RegexpRule{Base: "/", To: "/to", Regexp: regexp.MustCompile(".*")}, + &rewrite.ComplexRule{Base: "/", To: "/to", Regexp: regexp.MustCompile(".*")}, }}, {`rewrite { regexp .* to /to ext / html txt }`, false, []rewrite.Rule{ - &rewrite.RegexpRule{Base: "/", To: "/to", Exts: []string{"/", "html", "txt"}, Regexp: regexp.MustCompile(".*")}, + &rewrite.ComplexRule{Base: "/", To: "/to", Exts: []string{"/", "html", "txt"}, Regexp: regexp.MustCompile(".*")}, }}, {`rewrite /path { r rr @@ -116,26 +116,26 @@ func TestRewriteParse(t *testing.T) { to /to } `, false, []rewrite.Rule{ - &rewrite.RegexpRule{Base: "/path", To: "/dest", Regexp: regexp.MustCompile("rr")}, - &rewrite.RegexpRule{Base: "/", To: "/to", Regexp: regexp.MustCompile("[a-z]+")}, + &rewrite.ComplexRule{Base: "/path", To: "/dest", Regexp: regexp.MustCompile("rr")}, + &rewrite.ComplexRule{Base: "/", To: "/to", Regexp: regexp.MustCompile("[a-z]+")}, }}, {`rewrite { to /to }`, true, []rewrite.Rule{ - &rewrite.RegexpRule{}, + &rewrite.ComplexRule{}, }}, {`rewrite { r .* }`, true, []rewrite.Rule{ - &rewrite.RegexpRule{}, + &rewrite.ComplexRule{}, }}, {`rewrite { }`, true, []rewrite.Rule{ - &rewrite.RegexpRule{}, + &rewrite.ComplexRule{}, }}, {`rewrite /`, true, []rewrite.Rule{ - &rewrite.RegexpRule{}, + &rewrite.ComplexRule{}, }}, } @@ -157,8 +157,8 @@ func TestRewriteParse(t *testing.T) { } for j, e := range test.expected { - actualRule := actual[j].(*rewrite.RegexpRule) - expectedRule := e.(*rewrite.RegexpRule) + actualRule := actual[j].(*rewrite.ComplexRule) + expectedRule := e.(*rewrite.ComplexRule) if actualRule.Base != expectedRule.Base { t.Errorf("Test %d, rule %d: Expected Base=%s, got %s", diff --git a/middleware/rewrite/condition.go b/middleware/rewrite/condition.go new file mode 100644 index 00000000..ab69ef4a --- /dev/null +++ b/middleware/rewrite/condition.go @@ -0,0 +1,110 @@ +package rewrite + +import ( + "fmt" + "github.com/mholt/caddy/middleware" + "net/http" + "regexp" + "strings" +) + +const ( + // Operators + Is = "is" + Not = "not" + Has = "has" + StartsWith = "starts_with" + EndsWith = "ends_with" + Match = "match" +) + +func operatorError(operator string) error { + return fmt.Errorf("Invalid operator", operator) +} + +func newReplacer(r *http.Request) middleware.Replacer { + return middleware.NewReplacer(r, nil, "") +} + +// condition is a rewrite condition. +type condition func(string, string) bool + +var conditions = map[string]condition{ + Is: isFunc, + Not: notFunc, + Has: hasFunc, + StartsWith: startsWithFunc, + EndsWith: endsWithFunc, + Match: matchFunc, +} + +// isFunc is condition for Is operator. +// It checks for equality. +func isFunc(a, b string) bool { + return a == b +} + +// notFunc is condition for Not operator. +// It checks for inequality. +func notFunc(a, b string) bool { + return a != b +} + +// hasFunc is condition for Has operator. +// It checks if b is a substring of a. +func hasFunc(a, b string) bool { + return strings.Contains(a, b) +} + +// startsWithFunc is condition for StartsWith operator. +// It checks if b is a prefix of a. +func startsWithFunc(a, b string) bool { + return strings.HasPrefix(a, b) +} + +// endsWithFunc is condition for EndsWith operator. +// It checks if b is a suffix of a. +func endsWithFunc(a, b string) bool { + return strings.HasSuffix(a, b) +} + +// matchFunc is condition for Match operator. +// It does regexp matching of +func matchFunc(a, b string) bool { + matched, _ := regexp.MatchString(b, a) + return matched +} + +// If is statement for a rewrite condition. +type If struct { + A string + Operator string + B string +} + +// True returns true if the condition is true and false otherwise. +// If r is not nil, it replaces placeholders before comparison. +func (i If) True(r *http.Request) bool { + if c, ok := conditions[i.Operator]; ok { + a, b := i.A, i.B + if r != nil { + replacer := newReplacer(r) + a = replacer.Replace(i.A) + b = replacer.Replace(i.B) + } + return c(a, b) + } + return false +} + +// NewIf creates a new If condition. +func NewIf(a, operator, b string) (If, error) { + if _, ok := conditions[operator]; !ok { + return If{}, operatorError(operator) + } + return If{ + A: a, + Operator: operator, + B: b, + }, nil +} diff --git a/middleware/rewrite/rewrite.go b/middleware/rewrite/rewrite.go index 88944f73..cc60e82c 100644 --- a/middleware/rewrite/rewrite.go +++ b/middleware/rewrite/rewrite.go @@ -5,7 +5,6 @@ package rewrite import ( "fmt" "net/http" - "net/url" "path" "path/filepath" "regexp" @@ -16,14 +15,15 @@ import ( // Rewrite is middleware to rewrite request locations internally before being handled. type Rewrite struct { - Next middleware.Handler - Rules []Rule + Next middleware.Handler + FileSys http.FileSystem + Rules []Rule } // ServeHTTP implements the middleware.Handler interface. func (rw Rewrite) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) { for _, rule := range rw.Rules { - if ok := rule.Rewrite(r); ok { + if ok := rule.Rewrite(rw.FileSys, r); ok { break } } @@ -33,7 +33,7 @@ func (rw Rewrite) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) // Rule describes an internal location rewrite rule. type Rule interface { // Rewrite rewrites the internal location of the current request. - Rewrite(*http.Request) bool + Rewrite(http.FileSystem, *http.Request) bool } // SimpleRule is a simple rewrite rule. @@ -47,23 +47,20 @@ func NewSimpleRule(from, to string) SimpleRule { } // Rewrite rewrites the internal location of the current request. -func (s SimpleRule) Rewrite(r *http.Request) bool { +func (s SimpleRule) Rewrite(fs http.FileSystem, r *http.Request) bool { if s.From == r.URL.Path { // take note of this rewrite for internal use by fastcgi // all we need is the URI, not full URL r.Header.Set(headerFieldName, r.URL.RequestURI()) - // replace variables - to := path.Clean(middleware.NewReplacer(r, nil, "").Replace(s.To)) - - r.URL.Path = to - return true + // attempt rewrite + return To(fs, r, s.To) } return false } -// RegexpRule is a rewrite rule based on a regular expression -type RegexpRule struct { +// ComplexRule is a rewrite rule based on a regular expression +type ComplexRule struct { // Path base. Request to this path and subpaths will be rewritten Base string @@ -73,18 +70,26 @@ type RegexpRule struct { // Extensions to filter by Exts []string + // Rewrite conditions + Ifs []If + *regexp.Regexp } // NewRegexpRule creates a new RegexpRule. It returns an error if regexp // pattern (pattern) or extensions (ext) are invalid. -func NewRegexpRule(base, pattern, to string, ext []string) (*RegexpRule, error) { - r, err := regexp.Compile(pattern) - if err != nil { - return nil, err +func NewComplexRule(base, pattern, to string, ext []string, ifs []If) (*ComplexRule, error) { + // validate regexp if present + var r *regexp.Regexp + if pattern != "" { + var err error + r, err = regexp.Compile(pattern) + if err != nil { + return nil, err + } } - // validate extensions + // validate extensions if present for _, v := range ext { if len(v) < 2 || (len(v) < 3 && v[0] == '!') { // check if no extension is specified @@ -94,16 +99,17 @@ func NewRegexpRule(base, pattern, to string, ext []string) (*RegexpRule, error) } } - return &RegexpRule{ - base, - to, - ext, - r, + return &ComplexRule{ + Base: base, + To: to, + Exts: ext, + Ifs: ifs, + Regexp: r, }, nil } // Rewrite rewrites the internal location of the current request. -func (r *RegexpRule) Rewrite(req *http.Request) bool { +func (r *ComplexRule) Rewrite(fs http.FileSystem, req *http.Request) bool { rPath := req.URL.Path // validate base @@ -127,31 +133,13 @@ func (r *RegexpRule) Rewrite(req *http.Request) bool { return false } - // replace variables - to := path.Clean(middleware.NewReplacer(req, nil, "").Replace(r.To)) - - // validate resulting path - url, err := url.Parse(to) - if err != nil { - return false - } - - // take note of this rewrite for internal use by fastcgi - // all we need is the URI, not full URL - req.Header.Set(headerFieldName, req.URL.RequestURI()) - - // perform rewrite - req.URL.Path = url.Path - if url.RawQuery != "" { - // overwrite query string if present - req.URL.RawQuery = url.RawQuery - } - return true + // attempt rewrite + return To(fs, req, r.To) } // matchExt matches rPath against registered file extensions. // Returns true if a match is found and false otherwise. -func (r *RegexpRule) matchExt(rPath string) bool { +func (r *ComplexRule) matchExt(rPath string) bool { f := filepath.Base(rPath) ext := path.Ext(f) if ext == "" { diff --git a/middleware/rewrite/rewrite_test.go b/middleware/rewrite/rewrite_test.go index fb047026..ca4cc512 100644 --- a/middleware/rewrite/rewrite_test.go +++ b/middleware/rewrite/rewrite_test.go @@ -4,9 +4,8 @@ import ( "fmt" "net/http" "net/http/httptest" - "testing" - "strings" + "testing" "github.com/mholt/caddy/middleware" ) @@ -38,7 +37,7 @@ func TestRewrite(t *testing.T) { if s := strings.Split(regexpRule[3], "|"); len(s) > 1 { ext = s[:len(s)-1] } - rule, err := NewRegexpRule(regexpRule[0], regexpRule[1], regexpRule[2], ext) + rule, err := NewComplexRule(regexpRule[0], regexpRule[1], regexpRule[2], ext) if err != nil { t.Fatal(err) } diff --git a/middleware/rewrite/to.go b/middleware/rewrite/to.go new file mode 100644 index 00000000..0ee8a159 --- /dev/null +++ b/middleware/rewrite/to.go @@ -0,0 +1,68 @@ +package rewrite + +import ( + "log" + "net/http" + "net/url" + "strings" +) + +// To attempts rewrite. It attempts to rewrite to first valid path +// or the last path if none of the paths are valid. +// Returns true if rewrite is successful and false otherwise. +func To(fs http.FileSystem, r *http.Request, to string) bool { + tos := strings.Fields(to) + replacer := newReplacer(r) + + // try each rewrite paths + t := "" + for _, v := range tos { + t = replacer.Replace(v) + if isValidFile(fs, t) { + break + } + } + + // validate resulting path + u, err := url.Parse(t) + if err != nil { + // Let the user know we got here. Rewrite is expected but + // the resulting url is invalid. + log.Printf("[ERROR] rewrite: resulting path '%v' is invalid. error: %v", t, err) + return false + } + + // take note of this rewrite for internal use by fastcgi + // all we need is the URI, not full URL + r.Header.Set(headerFieldName, r.URL.RequestURI()) + + // perform rewrite + r.URL.Path = u.Path + if u.RawQuery != "" { + // overwrite query string if present + r.URL.RawQuery = u.RawQuery + } + if u.Fragment != "" { + // overwrite fragment if present + r.URL.Fragment = u.Fragment + } + + return true +} + +// isValidFile checks if file exists on the filesystem. +// if file ends with `/`, it is validated as a directory. +func isValidFile(fs http.FileSystem, file string) bool { + f, err := fs.Open(file) + if err != nil { + return false + } + defer f.Close() + + stat, err := f.Stat() + if err != nil { + return false + } + + return strings.HasSuffix(file, "/") && stat.IsDir() +} From 4d5bc9fa6c2589a086f7033658c265a66ae7705c Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Wed, 23 Dec 2015 09:02:52 +0100 Subject: [PATCH 35/44] Backward compatibility ensured. --- caddy/setup/rewrite.go | 4 ++-- caddy/setup/rewrite_test.go | 5 ----- middleware/rewrite/rewrite.go | 15 ++++++++++++--- middleware/rewrite/rewrite_test.go | 3 ++- middleware/rewrite/to.go | 3 ++- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/caddy/setup/rewrite.go b/caddy/setup/rewrite.go index c70a2f94..e642d071 100644 --- a/caddy/setup/rewrite.go +++ b/caddy/setup/rewrite.go @@ -78,8 +78,8 @@ func rewriteParse(c *Controller) ([]rewrite.Rule, error) { return nil, c.ArgErr() } } - // ensure pattern and to are specified - if pattern == "" || to == "" { + // ensure to is specified + if to == "" { return nil, c.ArgErr() } if rule, err = rewrite.NewComplexRule(base, pattern, to, ext, ifs); err != nil { diff --git a/caddy/setup/rewrite_test.go b/caddy/setup/rewrite_test.go index ddef7cd4..13c1372f 100644 --- a/caddy/setup/rewrite_test.go +++ b/caddy/setup/rewrite_test.go @@ -119,11 +119,6 @@ func TestRewriteParse(t *testing.T) { &rewrite.ComplexRule{Base: "/path", To: "/dest", Regexp: regexp.MustCompile("rr")}, &rewrite.ComplexRule{Base: "/", To: "/to", Regexp: regexp.MustCompile("[a-z]+")}, }}, - {`rewrite { - to /to - }`, true, []rewrite.Rule{ - &rewrite.ComplexRule{}, - }}, {`rewrite { r .* }`, true, []rewrite.Rule{ diff --git a/middleware/rewrite/rewrite.go b/middleware/rewrite/rewrite.go index cc60e82c..f173e754 100644 --- a/middleware/rewrite/rewrite.go +++ b/middleware/rewrite/rewrite.go @@ -128,9 +128,18 @@ func (r *ComplexRule) Rewrite(fs http.FileSystem, req *http.Request) bool { start-- } - // validate regexp - if !r.MatchString(rPath[start:]) { - return false + // validate regexp if present + if r.Regexp != nil { + if !r.MatchString(rPath[start:]) { + return false + } + } + + // validate rewrite conditions + for _, i := range r.Ifs { + if !i.True(req) { + return false + } } // attempt rewrite diff --git a/middleware/rewrite/rewrite_test.go b/middleware/rewrite/rewrite_test.go index ca4cc512..6230f0d9 100644 --- a/middleware/rewrite/rewrite_test.go +++ b/middleware/rewrite/rewrite_test.go @@ -18,6 +18,7 @@ func TestRewrite(t *testing.T) { NewSimpleRule("/a", "/b"), NewSimpleRule("/b", "/b{uri}"), }, + FileSys: http.Dir("."), } regexpRules := [][]string{ @@ -37,7 +38,7 @@ func TestRewrite(t *testing.T) { if s := strings.Split(regexpRule[3], "|"); len(s) > 1 { ext = s[:len(s)-1] } - rule, err := NewComplexRule(regexpRule[0], regexpRule[1], regexpRule[2], ext) + rule, err := NewComplexRule(regexpRule[0], regexpRule[1], regexpRule[2], ext, nil) if err != nil { t.Fatal(err) } diff --git a/middleware/rewrite/to.go b/middleware/rewrite/to.go index 0ee8a159..294b8f04 100644 --- a/middleware/rewrite/to.go +++ b/middleware/rewrite/to.go @@ -4,6 +4,7 @@ import ( "log" "net/http" "net/url" + "path" "strings" ) @@ -17,7 +18,7 @@ func To(fs http.FileSystem, r *http.Request, to string) bool { // try each rewrite paths t := "" for _, v := range tos { - t = replacer.Replace(v) + t = path.Clean(replacer.Replace(v)) if isValidFile(fs, t) { break } From 1ed786f836b59c7893bc944939cc520dcfe00208 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Wed, 23 Dec 2015 09:36:00 +0100 Subject: [PATCH 36/44] Cleanups and panic prevention in tests. --- caddy/setup/rewrite.go | 6 ++++-- middleware/rewrite/condition.go | 5 +++-- middleware/rewrite/to.go | 4 ++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/caddy/setup/rewrite.go b/caddy/setup/rewrite.go index e642d071..4c84cb5f 100644 --- a/caddy/setup/rewrite.go +++ b/caddy/setup/rewrite.go @@ -2,6 +2,7 @@ package setup import ( "net/http" + "strings" "github.com/mholt/caddy/middleware" "github.com/mholt/caddy/middleware/rewrite" @@ -54,10 +55,11 @@ func rewriteParse(c *Controller) ([]rewrite.Rule, error) { } pattern = c.Val() case "to": - if !c.NextArg() { + args1 := c.RemainingArgs() + if len(args1) == 0 { return nil, c.ArgErr() } - to = c.Val() + to = strings.Join(args1, " ") case "ext": args1 := c.RemainingArgs() if len(args1) == 0 { diff --git a/middleware/rewrite/condition.go b/middleware/rewrite/condition.go index ab69ef4a..51d4b3a2 100644 --- a/middleware/rewrite/condition.go +++ b/middleware/rewrite/condition.go @@ -2,10 +2,11 @@ package rewrite import ( "fmt" - "github.com/mholt/caddy/middleware" "net/http" "regexp" "strings" + + "github.com/mholt/caddy/middleware" ) const ( @@ -19,7 +20,7 @@ const ( ) func operatorError(operator string) error { - return fmt.Errorf("Invalid operator", operator) + return fmt.Errorf("Invalid operator %v", operator) } func newReplacer(r *http.Request) middleware.Replacer { diff --git a/middleware/rewrite/to.go b/middleware/rewrite/to.go index 294b8f04..1dc48fdb 100644 --- a/middleware/rewrite/to.go +++ b/middleware/rewrite/to.go @@ -54,6 +54,10 @@ func To(fs http.FileSystem, r *http.Request, to string) bool { // isValidFile checks if file exists on the filesystem. // if file ends with `/`, it is validated as a directory. func isValidFile(fs http.FileSystem, file string) bool { + if fs == nil { + return false + } + f, err := fs.Open(file) if err != nil { return false From 9110dc4745cf10940b0122ab749ceb992268e80e Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Wed, 23 Dec 2015 12:11:11 +0100 Subject: [PATCH 37/44] Refactor. Tests and tests data. --- caddy/setup/rewrite_test.go | 31 +++++--- middleware/rewrite/condition.go | 2 +- middleware/rewrite/condition_test.go | 90 +++++++++++++++++++++++ middleware/rewrite/rewrite_test.go | 4 +- middleware/rewrite/testdata/testdir/empty | 0 middleware/rewrite/testdata/testfile | 1 + middleware/rewrite/to.go | 15 +++- middleware/rewrite/to_test.go | 44 +++++++++++ 8 files changed, 174 insertions(+), 13 deletions(-) create mode 100644 middleware/rewrite/condition_test.go create mode 100644 middleware/rewrite/testdata/testdir/empty create mode 100644 middleware/rewrite/testdata/testfile create mode 100644 middleware/rewrite/to_test.go diff --git a/caddy/setup/rewrite_test.go b/caddy/setup/rewrite_test.go index 13c1372f..f3d2e925 100644 --- a/caddy/setup/rewrite_test.go +++ b/caddy/setup/rewrite_test.go @@ -1,10 +1,9 @@ package setup import ( - "testing" - "fmt" "regexp" + "testing" "github.com/mholt/caddy/middleware/rewrite" ) @@ -96,9 +95,9 @@ func TestRewriteParse(t *testing.T) { }{ {`rewrite { r .* - to /to + to /to /index.php? }`, false, []rewrite.Rule{ - &rewrite.ComplexRule{Base: "/", To: "/to", Regexp: regexp.MustCompile(".*")}, + &rewrite.ComplexRule{Base: "/", To: "/to /index.php?", Regexp: regexp.MustCompile(".*")}, }}, {`rewrite { regexp .* @@ -113,11 +112,11 @@ func TestRewriteParse(t *testing.T) { } rewrite / { regexp [a-z]+ - to /to + to /to /to2 } `, false, []rewrite.Rule{ &rewrite.ComplexRule{Base: "/path", To: "/dest", Regexp: regexp.MustCompile("rr")}, - &rewrite.ComplexRule{Base: "/", To: "/to", Regexp: regexp.MustCompile("[a-z]+")}, + &rewrite.ComplexRule{Base: "/", To: "/to /to2", Regexp: regexp.MustCompile("[a-z]+")}, }}, {`rewrite { r .* @@ -132,6 +131,12 @@ func TestRewriteParse(t *testing.T) { {`rewrite /`, true, []rewrite.Rule{ &rewrite.ComplexRule{}, }}, + {`rewrite { + to /to + if {path} is a + }`, false, []rewrite.Rule{ + &rewrite.ComplexRule{Base: "/", To: "/to", Ifs: []rewrite.If{rewrite.If{"{path}", "is", "a"}}}, + }}, } for i, test := range regexpTests { @@ -170,10 +175,18 @@ func TestRewriteParse(t *testing.T) { i, j, expectedRule.To, actualRule.To) } - if actualRule.String() != expectedRule.String() { - t.Errorf("Test %d, rule %d: Expected Pattern=%s, got %s", - i, j, expectedRule.String(), actualRule.String()) + if actualRule.Regexp != nil { + if actualRule.String() != expectedRule.String() { + t.Errorf("Test %d, rule %d: Expected Pattern=%s, got %s", + i, j, expectedRule.String(), actualRule.String()) + } } + + if fmt.Sprint(actualRule.Ifs) != fmt.Sprint(expectedRule.Ifs) { + t.Errorf("Test %d, rule %d: Expected Pattern=%s, got %s", + i, j, fmt.Sprint(expectedRule.Ifs), fmt.Sprint(actualRule.Ifs)) + } + } } diff --git a/middleware/rewrite/condition.go b/middleware/rewrite/condition.go index 51d4b3a2..c863af4f 100644 --- a/middleware/rewrite/condition.go +++ b/middleware/rewrite/condition.go @@ -70,7 +70,7 @@ func endsWithFunc(a, b string) bool { } // matchFunc is condition for Match operator. -// It does regexp matching of +// It does regexp matching of a against pattern in b func matchFunc(a, b string) bool { matched, _ := regexp.MatchString(b, a) return matched diff --git a/middleware/rewrite/condition_test.go b/middleware/rewrite/condition_test.go new file mode 100644 index 00000000..7db20647 --- /dev/null +++ b/middleware/rewrite/condition_test.go @@ -0,0 +1,90 @@ +package rewrite + +import ( + "net/http" + "strings" + "testing" +) + +func TestConditions(t *testing.T) { + tests := []struct { + condition string + isTrue bool + }{ + {"a is b", false}, + {"a is a", true}, + {"a not b", true}, + {"a not a", false}, + {"a has a", true}, + {"a has b", false}, + {"ba has b", true}, + {"bab has b", true}, + {"bab has bb", false}, + {"bab starts_with bb", false}, + {"bab starts_with ba", true}, + {"bab starts_with bab", true}, + {"bab ends_with bb", false}, + {"bab ends_with bab", true}, + {"bab ends_with ab", true}, + {"a match *", false}, + {"a match a", true}, + {"a match .*", true}, + {"a match a.*", true}, + {"a match b.*", false}, + {"ba match b.*", true}, + {"ba match b[a-z]", true}, + {"b0 match b[a-z]", false}, + {"b0a match b[a-z]", false}, + {"b0a match b[a-z]+", false}, + {"b0a match b[a-z0-9]+", true}, + } + + for i, test := range tests { + str := strings.Fields(test.condition) + ifCond, err := NewIf(str[0], str[1], str[2]) + if err != nil { + t.Error(err) + } + isTrue := ifCond.True(nil) + if isTrue != test.isTrue { + t.Errorf("Test %v: expected %v found %v", i, test.isTrue, isTrue) + } + } + + invalidOperators := []string{"ss", "and", "if"} + for _, op := range invalidOperators { + _, err := NewIf("a", op, "b") + if err == nil { + t.Error("Invalid operator %v used, expected error.", op) + } + } + + replaceTests := []struct { + url string + condition string + isTrue bool + }{ + {"/home", "{uri} match /home", true}, + {"/hom", "{uri} match /home", false}, + {"/hom", "{uri} starts_with /home", false}, + {"/hom", "{uri} starts_with /h", true}, + {"/home/.hiddenfile", `{uri} match \/\.(.*)`, true}, + {"/home/.hiddendir/afile", `{uri} match \/\.(.*)`, true}, + } + + for i, test := range replaceTests { + r, err := http.NewRequest("GET", test.url, nil) + if err != nil { + t.Error(err) + } + str := strings.Fields(test.condition) + ifCond, err := NewIf(str[0], str[1], str[2]) + if err != nil { + t.Error(err) + } + isTrue := ifCond.True(r) + if isTrue != test.isTrue { + t.Errorf("Test %v: expected %v found %v", i, test.isTrue, isTrue) + } + } +} diff --git a/middleware/rewrite/rewrite_test.go b/middleware/rewrite/rewrite_test.go index 6230f0d9..a538b79d 100644 --- a/middleware/rewrite/rewrite_test.go +++ b/middleware/rewrite/rewrite_test.go @@ -21,7 +21,7 @@ func TestRewrite(t *testing.T) { FileSys: http.Dir("."), } - regexpRules := [][]string{ + regexps := [][]string{ {"/reg/", ".*", "/to", ""}, {"/r/", "[a-z]+", "/toaz", "!.html|"}, {"/url/", "a([a-z0-9]*)s([A-Z]{2})", "/to/{path}", ""}, @@ -33,7 +33,7 @@ func TestRewrite(t *testing.T) { {"/ab/", `.*\.jpg`, "/ajpg", ""}, } - for _, regexpRule := range regexpRules { + for _, regexpRule := range regexps { var ext []string if s := strings.Split(regexpRule[3], "|"); len(s) > 1 { ext = s[:len(s)-1] diff --git a/middleware/rewrite/testdata/testdir/empty b/middleware/rewrite/testdata/testdir/empty new file mode 100644 index 00000000..e69de29b diff --git a/middleware/rewrite/testdata/testfile b/middleware/rewrite/testdata/testfile new file mode 100644 index 00000000..7b4d68d7 --- /dev/null +++ b/middleware/rewrite/testdata/testfile @@ -0,0 +1 @@ +empty \ No newline at end of file diff --git a/middleware/rewrite/to.go b/middleware/rewrite/to.go index 1dc48fdb..d6c7f521 100644 --- a/middleware/rewrite/to.go +++ b/middleware/rewrite/to.go @@ -19,6 +19,13 @@ func To(fs http.FileSystem, r *http.Request, to string) bool { t := "" for _, v := range tos { t = path.Clean(replacer.Replace(v)) + + // add trailing slash for directories, if present + if strings.HasSuffix(v, "/") && !strings.HasSuffix(t, "/") { + t += "/" + } + + // validate file if isValidFile(fs, t) { break } @@ -69,5 +76,11 @@ func isValidFile(fs http.FileSystem, file string) bool { return false } - return strings.HasSuffix(file, "/") && stat.IsDir() + // directory + if strings.HasSuffix(file, "/") { + return stat.IsDir() + } + + // file + return !stat.IsDir() } diff --git a/middleware/rewrite/to_test.go b/middleware/rewrite/to_test.go new file mode 100644 index 00000000..2d8b535a --- /dev/null +++ b/middleware/rewrite/to_test.go @@ -0,0 +1,44 @@ +package rewrite + +import ( + "net/http" + "net/url" + "testing" +) + +func TestTo(t *testing.T) { + fs := http.Dir("testdata") + tests := []struct { + url string + to string + expected string + }{ + {"/", "/somefiles", "/somefiles"}, + {"/somefiles", "/somefiles /index.php{uri}", "/index.php/somefiles"}, + {"/somefiles", "/testfile /index.php{uri}", "/testfile"}, + {"/somefiles", "/testfile/ /index.php{uri}", "/index.php/somefiles"}, + {"/somefiles", "/somefiles /index.php{uri}", "/index.php/somefiles"}, + {"/?a=b", "/somefiles /index.php?{query}", "/index.php?a=b"}, + {"/?a=b", "/testfile /index.php?{query}", "/testfile?a=b"}, + {"/?a=b", "/testdir /index.php?{query}", "/index.php?a=b"}, + {"/?a=b", "/testdir/ /index.php?{query}", "/testdir/?a=b"}, + } + + uri := func(r *url.URL) string { + uri := r.Path + if r.RawQuery != "" { + uri += "?" + r.RawQuery + } + return uri + } + for i, test := range tests { + r, err := http.NewRequest("GET", test.url, nil) + if err != nil { + t.Error(err) + } + To(fs, r, test.to) + if uri(r.URL) != test.expected { + t.Errorf("Test %v: expected %v found %v", i, test.expected, uri(r.URL)) + } + } +} From 92bd91441893e41c39bb9ca2cf5d2dc509bfdbc8 Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Wed, 23 Dec 2015 13:23:43 +0100 Subject: [PATCH 38/44] Fix vet errors. --- caddy/setup/rewrite_test.go | 2 +- middleware/rewrite/condition_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/caddy/setup/rewrite_test.go b/caddy/setup/rewrite_test.go index f3d2e925..c43818b2 100644 --- a/caddy/setup/rewrite_test.go +++ b/caddy/setup/rewrite_test.go @@ -135,7 +135,7 @@ func TestRewriteParse(t *testing.T) { to /to if {path} is a }`, false, []rewrite.Rule{ - &rewrite.ComplexRule{Base: "/", To: "/to", Ifs: []rewrite.If{rewrite.If{"{path}", "is", "a"}}}, + &rewrite.ComplexRule{Base: "/", To: "/to", Ifs: []rewrite.If{rewrite.If{A: "{path}", Operator: "is", B: "a"}}}, }}, } diff --git a/middleware/rewrite/condition_test.go b/middleware/rewrite/condition_test.go index 7db20647..c056f896 100644 --- a/middleware/rewrite/condition_test.go +++ b/middleware/rewrite/condition_test.go @@ -55,7 +55,7 @@ func TestConditions(t *testing.T) { for _, op := range invalidOperators { _, err := NewIf("a", op, "b") if err == nil { - t.Error("Invalid operator %v used, expected error.", op) + t.Errorf("Invalid operator %v used, expected error.", op) } } From 168723a026cdad53e2d5d8434003fa3e57f04e9a Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Thu, 24 Dec 2015 09:00:10 +0100 Subject: [PATCH 39/44] Added escaped versions of uri, query and path. --- middleware/replacer.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/middleware/replacer.go b/middleware/replacer.go index 29c695b7..3fb389a1 100644 --- a/middleware/replacer.go +++ b/middleware/replacer.go @@ -3,6 +3,7 @@ package middleware import ( "net" "net/http" + "net/url" "path" "strconv" "strings" @@ -38,11 +39,13 @@ func NewReplacer(r *http.Request, rr *responseRecorder, emptyValue string) Repla } return "http" }(), - "{host}": r.Host, - "{path}": r.URL.Path, - "{query}": r.URL.RawQuery, - "{fragment}": r.URL.Fragment, - "{proto}": r.Proto, + "{host}": r.Host, + "{path}": r.URL.Path, + "{path_escaped}": url.QueryEscape(r.URL.Path), + "{query}": r.URL.RawQuery, + "{query_escaped}": url.QueryEscape(r.URL.RawQuery), + "{fragment}": r.URL.Fragment, + "{proto}": r.Proto, "{remote}": func() string { if fwdFor := r.Header.Get("X-Forwarded-For"); fwdFor != "" { return fwdFor @@ -60,7 +63,8 @@ func NewReplacer(r *http.Request, rr *responseRecorder, emptyValue string) Repla } return port }(), - "{uri}": r.URL.RequestURI(), + "{uri}": r.URL.RequestURI(), + "{uri_escaped}": url.QueryEscape(r.URL.RequestURI()), "{when}": func() string { return time.Now().Format(timeFormat) }(), From 1e7ec3397b8e67d9e0f0879a2238b556b7d51988 Mon Sep 17 00:00:00 2001 From: Radim Marek Date: Tue, 29 Dec 2015 23:32:59 +0100 Subject: [PATCH 40/44] Import allows only one expression --- caddy/parse/parsing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caddy/parse/parsing.go b/caddy/parse/parsing.go index db4b7e42..6ef908b0 100644 --- a/caddy/parse/parsing.go +++ b/caddy/parse/parsing.go @@ -187,7 +187,7 @@ func (p *parser) doImport() error { } importPattern := p.Val() if p.NextArg() { - return p.Err("Import allows only one file to import") + return p.Err("Import allows only one expression, either file or glob pattern") } matches, err := filepath.Glob(importPattern) From 7dadcd58341b8ad63893a4669913099edbcf937c Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Wed, 30 Dec 2015 20:42:03 +0100 Subject: [PATCH 41/44] Add ability to set custom values. --- middleware/replacer.go | 6 ++++++ middleware/replacer_test.go | 40 +++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/middleware/replacer.go b/middleware/replacer.go index 3fb389a1..193fd4fc 100644 --- a/middleware/replacer.go +++ b/middleware/replacer.go @@ -16,6 +16,7 @@ import ( // NewReplacer to get one of these. type Replacer interface { Replace(string) string + Set(key, value string) } type replacer struct { @@ -117,6 +118,11 @@ func (r replacer) Replace(s string) string { return s } +// Set sets key to value in the replacements map. +func (r replacer) Set(key, value string) { + r.replacements["{"+key+"}"] = value +} + const ( timeFormat = "02/Jan/2006:15:04:05 -0700" headerReplacer = "{>" diff --git a/middleware/replacer_test.go b/middleware/replacer_test.go index 39d91a22..8f1147de 100644 --- a/middleware/replacer_test.go +++ b/middleware/replacer_test.go @@ -69,3 +69,43 @@ func TestReplace(t *testing.T) { } } + +func TestSet(t *testing.T) { + w := httptest.NewRecorder() + recordRequest := NewResponseRecorder(w) + userJSON := `{"username": "dennis"}` + + reader := strings.NewReader(userJSON) //Convert string to reader + + request, err := http.NewRequest("POST", "http://caddyserver.com", reader) //Create request with JSON body + if err != nil { + t.Fatalf("Request Formation Failed \n") + } + replaceValues := NewReplacer(request, recordRequest, "") + + replaceValues.Set("host", "getcaddy.com") + replaceValues.Set("method", "GET") + replaceValues.Set("status", "201") + replaceValues.Set("variable", "value") + + switch v := replaceValues.(type) { + case replacer: + + if v.Replace("This host is {host}") != "This host is getcaddy.com" { + t.Errorf("Expected host replacement failed") + } + if v.Replace("This request method is {method}") != "This request method is GET" { + t.Errorf("Expected method replacement failed") + } + if v.Replace("The response status is {status}") != "The response status is 201" { + t.Errorf("Expected status replacement failed") + } + if v.Replace("The value of variable is {variable}") != "The value of variable is value" { + t.Errorf("Expected status replacement failed") + } + + default: + t.Fatalf("Return Value from New Replacer expected pass type assertion into a replacer type \n") + } + +} From 3c086fb2e67b885d7c3a6bd417b0e2d6fc66074f Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Wed, 30 Dec 2015 21:47:37 +0100 Subject: [PATCH 42/44] Support for rewrite match group. --- middleware/rewrite/rewrite.go | 15 ++++++++++++--- middleware/rewrite/rewrite_test.go | 9 +++++++++ middleware/rewrite/to.go | 5 +++-- middleware/rewrite/to_test.go | 2 +- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/middleware/rewrite/rewrite.go b/middleware/rewrite/rewrite.go index f173e754..60cc0b9d 100644 --- a/middleware/rewrite/rewrite.go +++ b/middleware/rewrite/rewrite.go @@ -54,7 +54,7 @@ func (s SimpleRule) Rewrite(fs http.FileSystem, r *http.Request) bool { r.Header.Set(headerFieldName, r.URL.RequestURI()) // attempt rewrite - return To(fs, r, s.To) + return To(fs, r, s.To, newReplacer(r)) } return false } @@ -111,6 +111,7 @@ func NewComplexRule(base, pattern, to string, ext []string, ifs []If) (*ComplexR // Rewrite rewrites the internal location of the current request. func (r *ComplexRule) Rewrite(fs http.FileSystem, req *http.Request) bool { rPath := req.URL.Path + replacer := newReplacer(req) // validate base if !middleware.Path(rPath).Matches(r.Base) { @@ -130,8 +131,16 @@ func (r *ComplexRule) Rewrite(fs http.FileSystem, req *http.Request) bool { // validate regexp if present if r.Regexp != nil { - if !r.MatchString(rPath[start:]) { + matches := r.FindStringSubmatch(rPath[start:]) + switch len(matches) { + case 0: + // no match return false + default: + // set regexp match variables {1}, {2} ... + for i := 1; i < len(matches); i++ { + replacer.Set(fmt.Sprint(i), matches[i]) + } } } @@ -143,7 +152,7 @@ func (r *ComplexRule) Rewrite(fs http.FileSystem, req *http.Request) bool { } // attempt rewrite - return To(fs, req, r.To) + return To(fs, req, r.To, replacer) } // matchExt matches rPath against registered file extensions. diff --git a/middleware/rewrite/rewrite_test.go b/middleware/rewrite/rewrite_test.go index a538b79d..5b891606 100644 --- a/middleware/rewrite/rewrite_test.go +++ b/middleware/rewrite/rewrite_test.go @@ -31,6 +31,9 @@ func TestRewrite(t *testing.T) { {"/abcd/", "ab", "/a/{dir}/{file}", ".html|"}, {"/abcde/", "ab", "/a#{fragment}", ".html|"}, {"/ab/", `.*\.jpg`, "/ajpg", ""}, + {"/reggrp", `/ad/([0-9]+)([a-z]*)`, "/a{1}/{2}", ""}, + {"/reg2grp", `(.*)`, "/{1}", ""}, + {"/reg3grp", `(.*)/(.*)/(.*)`, "/{1}{2}{3}", ""}, } for _, regexpRule := range regexps { @@ -81,6 +84,12 @@ func TestRewrite(t *testing.T) { {"/abcde/abcde.html", "/a"}, {"/abcde/abcde.html#1234", "/a#1234"}, {"/ab/ab.jpg", "/ajpg"}, + {"/reggrp/ad/12", "/a12"}, + {"/reggrp/ad/124a", "/a124/a"}, + {"/reggrp/ad/124abc", "/a124/abc"}, + {"/reg2grp/ad/124abc", "/ad/124abc"}, + {"/reg3grp/ad/aa/66", "/adaa66"}, + {"/reg3grp/ad612/n1n/ab", "/ad612n1nab"}, } for i, test := range tests { diff --git a/middleware/rewrite/to.go b/middleware/rewrite/to.go index d6c7f521..8a577c5e 100644 --- a/middleware/rewrite/to.go +++ b/middleware/rewrite/to.go @@ -6,14 +6,15 @@ import ( "net/url" "path" "strings" + + "github.com/mholt/caddy/middleware" ) // To attempts rewrite. It attempts to rewrite to first valid path // or the last path if none of the paths are valid. // Returns true if rewrite is successful and false otherwise. -func To(fs http.FileSystem, r *http.Request, to string) bool { +func To(fs http.FileSystem, r *http.Request, to string, replacer middleware.Replacer) bool { tos := strings.Fields(to) - replacer := newReplacer(r) // try each rewrite paths t := "" diff --git a/middleware/rewrite/to_test.go b/middleware/rewrite/to_test.go index 2d8b535a..6133c0b6 100644 --- a/middleware/rewrite/to_test.go +++ b/middleware/rewrite/to_test.go @@ -36,7 +36,7 @@ func TestTo(t *testing.T) { if err != nil { t.Error(err) } - To(fs, r, test.to) + To(fs, r, test.to, newReplacer(r)) if uri(r.URL) != test.expected { t.Errorf("Test %v: expected %v found %v", i, test.expected, uri(r.URL)) } From e2a3ec4c3dbc537d10ed872b31c2ed693740b3fa Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 31 Dec 2015 12:12:16 -0700 Subject: [PATCH 43/44] Replacer supports case-insensitive header placeholders (fixes #476) --- middleware/replacer.go | 43 ++++++++----- middleware/replacer_test.go | 120 ++++++++++++++++++------------------ 2 files changed, 90 insertions(+), 73 deletions(-) diff --git a/middleware/replacer.go b/middleware/replacer.go index 193fd4fc..10f0e35a 100644 --- a/middleware/replacer.go +++ b/middleware/replacer.go @@ -86,9 +86,9 @@ func NewReplacer(r *http.Request, rr *responseRecorder, emptyValue string) Repla rep.replacements["{latency}"] = time.Since(rr.start).String() } - // Header placeholders - for header, val := range r.Header { - rep.replacements[headerReplacer+header+"}"] = strings.Join(val, ",") + // Header placeholders (case-insensitive) + for header, values := range r.Header { + rep.replacements[headerReplacer+strings.ToLower(header)+"}"] = strings.Join(values, ",") } return rep @@ -97,6 +97,32 @@ func NewReplacer(r *http.Request, rr *responseRecorder, emptyValue string) Repla // Replace performs a replacement of values on s and returns // the string with the replaced values. func (r replacer) Replace(s string) string { + // Header replacements - these are case-insensitive, so we can't just use strings.Replace() + startPos := strings.Index(s, headerReplacer) + for startPos > -1 { + // carefully find end of placeholder + endOffset := strings.Index(s[startPos+1:], "}") + if endOffset == -1 { + startPos = strings.Index(s[startPos+len(headerReplacer):], headerReplacer) + continue + } + endPos := startPos + len(headerReplacer) + endOffset + + // look for replacement, case-insensitive + placeholder := strings.ToLower(s[startPos:endPos]) + replacement := r.replacements[placeholder] + if replacement == "" { + replacement = r.emptyValue + } + + // do the replacement manually + s = s[:startPos] + replacement + s[endPos:] + + // move to next one + startPos = strings.Index(s[endOffset:], headerReplacer) + } + + // Regular replacements - these are easier because they're case-sensitive for placeholder, replacement := range r.replacements { if replacement == "" { replacement = r.emptyValue @@ -104,17 +130,6 @@ func (r replacer) Replace(s string) string { s = strings.Replace(s, placeholder, replacement, -1) } - // Replace any header placeholders that weren't found - for strings.Contains(s, headerReplacer) { - idxStart := strings.Index(s, headerReplacer) - endOffset := idxStart + len(headerReplacer) - idxEnd := strings.Index(s[endOffset:], "}") - if idxEnd > -1 { - s = s[:idxStart] + r.emptyValue + s[endOffset+idxEnd+1:] - } else { - break - } - } return s } diff --git a/middleware/replacer_test.go b/middleware/replacer_test.go index 8f1147de..46d7b8f9 100644 --- a/middleware/replacer_test.go +++ b/middleware/replacer_test.go @@ -10,102 +10,104 @@ import ( func TestNewReplacer(t *testing.T) { w := httptest.NewRecorder() recordRequest := NewResponseRecorder(w) - userJSON := `{"username": "dennis"}` + reader := strings.NewReader(`{"username": "dennis"}`) - reader := strings.NewReader(userJSON) //Convert string to reader - - request, err := http.NewRequest("POST", "http://caddyserver.com", reader) //Create request with JSON body + request, err := http.NewRequest("POST", "http://localhost", reader) if err != nil { - t.Fatalf("Request Formation Failed \n") + t.Fatal("Request Formation Failed\n") } replaceValues := NewReplacer(request, recordRequest, "") switch v := replaceValues.(type) { case replacer: - if v.replacements["{host}"] != "caddyserver.com" { - t.Errorf("Expected host to be caddyserver.com") + if v.replacements["{host}"] != "localhost" { + t.Error("Expected host to be localhost") } if v.replacements["{method}"] != "POST" { - t.Errorf("Expected request method to be POST") + t.Error("Expected request method to be POST") } if v.replacements["{status}"] != "200" { - t.Errorf("Expected status to be 200") + t.Error("Expected status to be 200") } default: - t.Fatalf("Return Value from New Replacer expected pass type assertion into a replacer type \n") + t.Fatal("Return Value from New Replacer expected pass type assertion into a replacer type\n") } } func TestReplace(t *testing.T) { w := httptest.NewRecorder() recordRequest := NewResponseRecorder(w) - userJSON := `{"username": "dennis"}` + reader := strings.NewReader(`{"username": "dennis"}`) - reader := strings.NewReader(userJSON) //Convert string to reader - - request, err := http.NewRequest("POST", "http://caddyserver.com", reader) //Create request with JSON body + request, err := http.NewRequest("POST", "http://localhost", reader) if err != nil { - t.Fatalf("Request Formation Failed \n") + t.Fatal("Request Formation Failed\n") } - replaceValues := NewReplacer(request, recordRequest, "") + request.Header.Set("Custom", "fooBar") + repl := NewReplacer(request, recordRequest, "-") - switch v := replaceValues.(type) { - case replacer: - - if v.Replace("This host is {host}") != "This host is caddyserver.com" { - t.Errorf("Expected host replacement failed") - } - if v.Replace("This request method is {method}") != "This request method is POST" { - t.Errorf("Expected method replacement failed") - } - if v.Replace("The response status is {status}") != "The response status is 200" { - t.Errorf("Expected status replacement failed") - } - - default: - t.Fatalf("Return Value from New Replacer expected pass type assertion into a replacer type \n") + if expected, actual := "This host is localhost.", repl.Replace("This host is {host}."); expected != actual { + t.Errorf("{host} replacement: expected '%s', got '%s'", expected, actual) + } + if expected, actual := "This request method is POST.", repl.Replace("This request method is {method}."); expected != actual { + t.Errorf("{method} replacement: expected '%s', got '%s'", expected, actual) + } + if expected, actual := "The response status is 200.", repl.Replace("The response status is {status}."); expected != actual { + t.Errorf("{status} replacement: expected '%s', got '%s'", expected, actual) + } + if expected, actual := "The Custom header is fooBar.", repl.Replace("The Custom header is {>Custom}."); expected != actual { + t.Errorf("{>Custom} replacement: expected '%s', got '%s'", expected, actual) } + // Test header case-insensitivity + if expected, actual := "The cUsToM header is fooBar...", repl.Replace("The cUsToM header is {>cUsToM}..."); expected != actual { + t.Errorf("{>cUsToM} replacement: expected '%s', got '%s'", expected, actual) + } + + // Test non-existent header/value + if expected, actual := "The Non-Existent header is -.", repl.Replace("The Non-Existent header is {>Non-Existent}."); expected != actual { + t.Errorf("{>Non-Existent} replacement: expected '%s', got '%s'", expected, actual) + } + + // Test bad placeholder + if expected, actual := "Bad {host placeholder...", repl.Replace("Bad {host placeholder..."); expected != actual { + t.Errorf("bad placeholder: expected '%s', got '%s'", expected, actual) + } + + // Test bad header placeholder + if expected, actual := "Bad {>Custom placeholder", repl.Replace("Bad {>Custom placeholder"); expected != actual { + t.Errorf("bad header placeholder: expected '%s', got '%s'", expected, actual) + } } func TestSet(t *testing.T) { w := httptest.NewRecorder() recordRequest := NewResponseRecorder(w) - userJSON := `{"username": "dennis"}` + reader := strings.NewReader(`{"username": "dennis"}`) - reader := strings.NewReader(userJSON) //Convert string to reader - - request, err := http.NewRequest("POST", "http://caddyserver.com", reader) //Create request with JSON body + request, err := http.NewRequest("POST", "http://localhost", reader) if err != nil { t.Fatalf("Request Formation Failed \n") } - replaceValues := NewReplacer(request, recordRequest, "") + repl := NewReplacer(request, recordRequest, "") - replaceValues.Set("host", "getcaddy.com") - replaceValues.Set("method", "GET") - replaceValues.Set("status", "201") - replaceValues.Set("variable", "value") + repl.Set("host", "getcaddy.com") + repl.Set("method", "GET") + repl.Set("status", "201") + repl.Set("variable", "value") - switch v := replaceValues.(type) { - case replacer: - - if v.Replace("This host is {host}") != "This host is getcaddy.com" { - t.Errorf("Expected host replacement failed") - } - if v.Replace("This request method is {method}") != "This request method is GET" { - t.Errorf("Expected method replacement failed") - } - if v.Replace("The response status is {status}") != "The response status is 201" { - t.Errorf("Expected status replacement failed") - } - if v.Replace("The value of variable is {variable}") != "The value of variable is value" { - t.Errorf("Expected status replacement failed") - } - - default: - t.Fatalf("Return Value from New Replacer expected pass type assertion into a replacer type \n") + if repl.Replace("This host is {host}") != "This host is getcaddy.com" { + t.Error("Expected host replacement failed") + } + if repl.Replace("This request method is {method}") != "This request method is GET" { + t.Error("Expected method replacement failed") + } + if repl.Replace("The response status is {status}") != "The response status is 201" { + t.Error("Expected status replacement failed") + } + if repl.Replace("The value of variable is {variable}") != "The value of variable is value" { + t.Error("Expected variable replacement failed") } - } From b6326d402d3025084cce6b28b1299506aaaa838c Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 31 Dec 2015 12:31:30 -0700 Subject: [PATCH 44/44] Fix for case-insensitive header replacements (#476) --- middleware/replacer.go | 34 +++++++++++++--------------------- middleware/replacer_test.go | 17 ++++++++++++++--- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/middleware/replacer.go b/middleware/replacer.go index 10f0e35a..8a3d202a 100644 --- a/middleware/replacer.go +++ b/middleware/replacer.go @@ -98,28 +98,20 @@ func NewReplacer(r *http.Request, rr *responseRecorder, emptyValue string) Repla // the string with the replaced values. func (r replacer) Replace(s string) string { // Header replacements - these are case-insensitive, so we can't just use strings.Replace() - startPos := strings.Index(s, headerReplacer) - for startPos > -1 { - // carefully find end of placeholder - endOffset := strings.Index(s[startPos+1:], "}") - if endOffset == -1 { - startPos = strings.Index(s[startPos+len(headerReplacer):], headerReplacer) - continue + for strings.Contains(s, headerReplacer) { + idxStart := strings.Index(s, headerReplacer) + endOffset := idxStart + len(headerReplacer) + idxEnd := strings.Index(s[endOffset:], "}") + if idxEnd > -1 { + placeholder := strings.ToLower(s[idxStart : endOffset+idxEnd+1]) + replacement := r.replacements[placeholder] + if replacement == "" { + replacement = r.emptyValue + } + s = s[:idxStart] + replacement + s[endOffset+idxEnd+1:] + } else { + break } - endPos := startPos + len(headerReplacer) + endOffset - - // look for replacement, case-insensitive - placeholder := strings.ToLower(s[startPos:endPos]) - replacement := r.replacements[placeholder] - if replacement == "" { - replacement = r.emptyValue - } - - // do the replacement manually - s = s[:startPos] + replacement + s[endPos:] - - // move to next one - startPos = strings.Index(s[endOffset:], headerReplacer) } // Regular replacements - these are easier because they're case-sensitive diff --git a/middleware/replacer_test.go b/middleware/replacer_test.go index 46d7b8f9..d98bd2de 100644 --- a/middleware/replacer_test.go +++ b/middleware/replacer_test.go @@ -45,7 +45,8 @@ func TestReplace(t *testing.T) { if err != nil { t.Fatal("Request Formation Failed\n") } - request.Header.Set("Custom", "fooBar") + request.Header.Set("Custom", "foobarbaz") + request.Header.Set("ShorterVal", "1") repl := NewReplacer(request, recordRequest, "-") if expected, actual := "This host is localhost.", repl.Replace("This host is {host}."); expected != actual { @@ -57,12 +58,12 @@ func TestReplace(t *testing.T) { if expected, actual := "The response status is 200.", repl.Replace("The response status is {status}."); expected != actual { t.Errorf("{status} replacement: expected '%s', got '%s'", expected, actual) } - if expected, actual := "The Custom header is fooBar.", repl.Replace("The Custom header is {>Custom}."); expected != actual { + if expected, actual := "The Custom header is foobarbaz.", repl.Replace("The Custom header is {>Custom}."); expected != actual { t.Errorf("{>Custom} replacement: expected '%s', got '%s'", expected, actual) } // Test header case-insensitivity - if expected, actual := "The cUsToM header is fooBar...", repl.Replace("The cUsToM header is {>cUsToM}..."); expected != actual { + if expected, actual := "The cUsToM header is foobarbaz...", repl.Replace("The cUsToM header is {>cUsToM}..."); expected != actual { t.Errorf("{>cUsToM} replacement: expected '%s', got '%s'", expected, actual) } @@ -80,6 +81,16 @@ func TestReplace(t *testing.T) { if expected, actual := "Bad {>Custom placeholder", repl.Replace("Bad {>Custom placeholder"); expected != actual { t.Errorf("bad header placeholder: expected '%s', got '%s'", expected, actual) } + + // Test bad header placeholder with valid one later + if expected, actual := "Bad -", repl.Replace("Bad {>Custom placeholder {>ShorterVal}"); expected != actual { + t.Errorf("bad header placeholders: expected '%s', got '%s'", expected, actual) + } + + // Test shorter header value with multiple placeholders + if expected, actual := "Short value 1 then foobarbaz.", repl.Replace("Short value {>ShorterVal} then {>Custom}."); expected != actual { + t.Errorf("short value: expected '%s', got '%s'", expected, actual) + } } func TestSet(t *testing.T) {