diff --git a/caddy/setup/markdown.go b/caddy/setup/markdown.go index 27ac14a5..08a05567 100644 --- a/caddy/setup/markdown.go +++ b/caddy/setup/markdown.go @@ -2,9 +2,7 @@ package setup import ( "net/http" - "path" "path/filepath" - "strings" "github.com/mholt/caddy/middleware" "github.com/mholt/caddy/middleware/markdown" @@ -25,25 +23,6 @@ func Markdown(c *Controller) (middleware.Middleware, error) { IndexFiles: []string{"index.md"}, } - // Sweep the whole path at startup to at least generate link index, maybe generate static site - c.Startup = append(c.Startup, func() error { - for i := range mdconfigs { - cfg := mdconfigs[i] - - // Generate link index and static files (if enabled) - if err := markdown.GenerateStatic(md, cfg); err != nil { - return err - } - - // Watch file changes for static site generation if not in development mode. - if !cfg.Development { - markdown.Watch(md, cfg, markdown.DefaultInterval) - } - } - - return nil - }) - return func(next middleware.Handler) middleware.Handler { md.Next = next return md @@ -55,9 +34,9 @@ func markdownParse(c *Controller) ([]*markdown.Config, error) { for c.Next() { md := &markdown.Config{ - Renderer: blackfriday.HtmlRenderer(0, "", ""), - Templates: make(map[string]string), - StaticFiles: make(map[string]string), + Renderer: blackfriday.HtmlRenderer(0, "", ""), + Extensions: make(map[string]struct{}), + Templates: make(map[string]string), } // Get the path scope @@ -80,7 +59,9 @@ func markdownParse(c *Controller) ([]*markdown.Config, error) { // If no extensions were specified, assume some defaults if len(md.Extensions) == 0 { - md.Extensions = []string{".md", ".markdown", ".mdown"} + md.Extensions[".md"] = struct{}{} + md.Extensions[".markdown"] = struct{}{} + md.Extensions[".mdown"] = struct{}{} } mdconfigs = append(mdconfigs, md) @@ -92,11 +73,9 @@ func markdownParse(c *Controller) ([]*markdown.Config, error) { func loadParams(c *Controller, mdc *markdown.Config) error { switch c.Val() { case "ext": - exts := c.RemainingArgs() - if len(exts) == 0 { - return c.ArgErr() + for _, ext := range c.RemainingArgs() { + mdc.Extensions[ext] = struct{}{} } - mdc.Extensions = append(mdc.Extensions, exts...) return nil case "css": if !c.NextArg() { @@ -113,7 +92,7 @@ func loadParams(c *Controller, mdc *markdown.Config) error { case "template": tArgs := c.RemainingArgs() switch len(tArgs) { - case 0: + default: return c.ArgErr() case 1: if _, ok := mdc.Templates[markdown.DefaultTemplate]; ok { @@ -126,31 +105,7 @@ func loadParams(c *Controller, mdc *markdown.Config) error { fpath := filepath.ToSlash(filepath.Clean(c.Root + string(filepath.Separator) + tArgs[1])) mdc.Templates[tArgs[0]] = fpath return nil - default: - return c.ArgErr() } - case "sitegen": - if c.NextArg() { - mdc.StaticDir = path.Join(c.Root, c.Val()) - } else { - mdc.StaticDir = path.Join(c.Root, markdown.DefaultStaticDir) - } - if c.NextArg() { - // only 1 argument allowed - return c.ArgErr() - } - return nil - case "dev": - if c.NextArg() { - mdc.Development = strings.ToLower(c.Val()) == "true" - } else { - mdc.Development = true - } - if c.NextArg() { - // only 1 argument allowed - return c.ArgErr() - } - return nil default: return c.Err("Expected valid markdown configuration property") } diff --git a/caddy/setup/markdown_test.go b/caddy/setup/markdown_test.go index e562678e..18af6254 100644 --- a/caddy/setup/markdown_test.go +++ b/caddy/setup/markdown_test.go @@ -1,15 +1,9 @@ package setup import ( - "bytes" "fmt" - "io/ioutil" - "net/http" - "os" - "path/filepath" "testing" - "github.com/mholt/caddy/middleware" "github.com/mholt/caddy/middleware/markdown" ) @@ -37,84 +31,14 @@ func TestMarkdown(t *testing.T) { if myHandler.Configs[0].PathScope != "/blog" { t.Errorf("Expected /blog as the Path Scope") } - if fmt.Sprint(myHandler.Configs[0].Extensions) != fmt.Sprint([]string{".md", ".markdown", ".mdown"}) { - t.Errorf("Expected .md, .markdown, and .mdown as default extensions") + if len(myHandler.Configs[0].Extensions) != 3 { + t.Error("Expected 3 markdown extensions") } -} - -func TestMarkdownStaticGen(t *testing.T) { - c := NewTestController(`markdown /blog { - ext .md - template tpl_with_include.html - sitegen -}`) - - c.Root = "./testdata" - mid, err := Markdown(c) - - if err != nil { - t.Errorf("Expected no errors, got: %v", err) - } - - if mid == nil { - t.Fatal("Expected middleware, was nil instead") - } - - for _, start := range c.Startup { - err := start() - if err != nil { - t.Errorf("Startup error: %v", err) + for _, key := range []string{".md", ".markdown", ".mdown"} { + if ext, ok := myHandler.Configs[0].Extensions[key]; !ok { + t.Errorf("Expected extensions to contain %v", ext) } } - - next := middleware.HandlerFunc(func(w http.ResponseWriter, r *http.Request) (int, error) { - t.Fatalf("Next shouldn't be called") - return 0, nil - }) - hndlr := mid(next) - mkdwn, ok := hndlr.(markdown.Markdown) - if !ok { - t.Fatalf("Was expecting a markdown.Markdown but got %T", hndlr) - } - - expectedStaticFiles := map[string]string{"/blog/first_post.md": "testdata/generated_site/blog/first_post.md/index.html"} - if fmt.Sprint(expectedStaticFiles) != fmt.Sprint(mkdwn.Configs[0].StaticFiles) { - t.Fatalf("Test expected StaticFiles to be %s, but got %s", - fmt.Sprint(expectedStaticFiles), fmt.Sprint(mkdwn.Configs[0].StaticFiles)) - } - - filePath := "testdata/generated_site/blog/first_post.md/index.html" - if _, err := os.Stat(filePath); err != nil { - t.Fatalf("An error occured when getting the file information: %v", err) - } - - html, err := ioutil.ReadFile(filePath) - if err != nil { - t.Fatalf("An error occured when getting the file content: %v", err) - } - - expectedBody := []byte(` - - -first_post - - -

Header title

- -

Test h1

- - - -`) - - if !bytes.Equal(html, expectedBody) { - t.Fatalf("Expected file content: %s got: %s", string(expectedBody), string(html)) - } - - fp := filepath.Join(c.Root, markdown.DefaultStaticDir) - if err = os.RemoveAll(fp); err != nil { - t.Errorf("Error while removing the generated static files: %v", err) - } } func TestMarkdownParse(t *testing.T) { @@ -129,20 +53,23 @@ func TestMarkdownParse(t *testing.T) { css /resources/css/blog.css js /resources/js/blog.js }`, false, []markdown.Config{{ - PathScope: "/blog", - Extensions: []string{".md", ".txt"}, - Styles: []string{"/resources/css/blog.css"}, - Scripts: []string{"/resources/js/blog.js"}, + PathScope: "/blog", + Extensions: map[string]struct{}{ + ".md": struct{}{}, + ".txt": struct{}{}, + }, + Styles: []string{"/resources/css/blog.css"}, + Scripts: []string{"/resources/js/blog.js"}, }}}, {`markdown /blog { ext .md template tpl_with_include.html - sitegen }`, false, []markdown.Config{{ - PathScope: "/blog", - Extensions: []string{".md"}, - Templates: map[string]string{markdown.DefaultTemplate: "testdata/tpl_with_include.html"}, - StaticDir: markdown.DefaultStaticDir, + PathScope: "/blog", + Extensions: map[string]struct{}{ + ".md": struct{}{}, + }, + Templates: map[string]string{markdown.DefaultTemplate: "testdata/tpl_with_include.html"}, }}}, } for i, test := range tests { diff --git a/middleware/context.go b/middleware/context.go index 3facb953..c43aa293 100644 --- a/middleware/context.go +++ b/middleware/context.go @@ -184,7 +184,11 @@ func (c Context) Markdown(filename string) (string, error) { return "", err } renderer := blackfriday.HtmlRenderer(0, "", "") - extns := blackfriday.EXTENSION_TABLES | blackfriday.EXTENSION_FENCED_CODE | blackfriday.EXTENSION_STRIKETHROUGH | blackfriday.EXTENSION_DEFINITION_LISTS + extns := 0 + extns |= blackfriday.EXTENSION_TABLES + extns |= blackfriday.EXTENSION_FENCED_CODE + extns |= blackfriday.EXTENSION_STRIKETHROUGH + extns |= blackfriday.EXTENSION_DEFINITION_LISTS markdown := blackfriday.Markdown([]byte(body), renderer, extns) return string(markdown), nil diff --git a/middleware/markdown/generator.go b/middleware/markdown/generator.go deleted file mode 100644 index d218f22b..00000000 --- a/middleware/markdown/generator.go +++ /dev/null @@ -1,146 +0,0 @@ -package markdown - -import ( - "crypto/md5" - "encoding/hex" - "fmt" - "io/ioutil" - "net/http" - "net/url" - "os" - "path/filepath" - "strings" - "sync" - - "github.com/mholt/caddy/middleware" -) - -// GenerateStatic generate static files and link index from markdowns. -// It only generates static files if it is enabled (cfg.StaticDir -// must be set). -func GenerateStatic(md Markdown, cfg *Config) error { - // Generate links since they may be needed, even without sitegen. - generated, err := generateLinks(md, cfg) - if err != nil { - return err - } - - // No new file changes, return. - if !generated { - return nil - } - - // If static site generation is enabled, generate the site. - if cfg.StaticDir != "" { - if err := generateStaticHTML(md, cfg); err != nil { - return err - } - } - - return nil -} - -type linkGenerator struct { - gens map[*Config]*linkGen - sync.Mutex -} - -var generator = linkGenerator{gens: make(map[*Config]*linkGen)} - -// generateLinks generates links to all markdown files ordered by newest date. -// This blocks until link generation is done. When called by multiple goroutines, -// the first caller starts the generation and others only wait. -// It returns if generation is done and any error that occurred. -func generateLinks(md Markdown, cfg *Config) (bool, error) { - generator.Lock() - - // if link generator exists for config and running, wait. - if g, ok := generator.gens[cfg]; ok { - if g.started() { - g.addWaiter() - generator.Unlock() - g.Wait() - // another goroutine has done the generation. - return false, g.lastErr - } - } - - g := &linkGen{} - generator.gens[cfg] = g - generator.Unlock() - - generated := g.generateLinks(md, cfg) - g.discardWaiters() - return generated, g.lastErr -} - -// generateStaticHTML generates static HTML files from markdowns. -func generateStaticHTML(md Markdown, cfg *Config) error { - // If generated site already exists, clear it out - _, err := os.Stat(cfg.StaticDir) - if err == nil { - err := os.RemoveAll(cfg.StaticDir) - if err != nil { - return err - } - } - - fp := filepath.Join(md.Root, cfg.PathScope) - - return filepath.Walk(fp, func(path string, info os.FileInfo, err error) error { - for _, ext := range cfg.Extensions { - if !info.IsDir() && strings.HasSuffix(info.Name(), ext) { - // Load the file - body, err := ioutil.ReadFile(path) - if err != nil { - return err - } - - // Get the relative path as if it were a HTTP request, - // then prepend with "/" (like a real HTTP request) - reqPath, err := filepath.Rel(md.Root, path) - if err != nil { - return err - } - reqPath = filepath.ToSlash(reqPath) - reqPath = "/" + reqPath - - // Create empty requests and url to cater for template values. - req, _ := http.NewRequest("", "/", nil) - urlVar, _ := url.Parse("/") - - // Generate the static file - ctx := middleware.Context{Root: md.FileSys, Req: req, URL: urlVar} - _, err = md.Process(cfg, reqPath, body, ctx) - if err != nil { - return err - } - - break // don't try other file extensions - } - } - return nil - }) -} - -// computeDirHash computes an hash on static directory of c. -func computeDirHash(md Markdown, c *Config) (string, error) { - dir := filepath.Join(md.Root, c.PathScope) - if _, err := os.Stat(dir); err != nil { - return "", err - } - - hashString := "" - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if !info.IsDir() && c.IsValidExt(filepath.Ext(path)) { - hashString += fmt.Sprintf("%v%v%v%v", info.ModTime(), info.Name(), info.Size(), path) - } - return nil - }) - if err != nil { - return "", err - } - - sum := md5.Sum([]byte(hashString)) - return hex.EncodeToString(sum[:]), nil -} diff --git a/middleware/markdown/markdown.go b/middleware/markdown/markdown.go index 9f14ca8d..5e4a2cb5 100644 --- a/middleware/markdown/markdown.go +++ b/middleware/markdown/markdown.go @@ -7,8 +7,7 @@ import ( "log" "net/http" "os" - "strings" - "sync" + "path" "github.com/mholt/caddy/middleware" "github.com/russross/blackfriday" @@ -52,7 +51,7 @@ type Config struct { PathScope string // List of extensions to consider as markdown files - Extensions []string + Extensions map[string]struct{} // List of style sheets to load for each markdown file Styles []string @@ -62,34 +61,6 @@ type Config struct { // Map of registered templates Templates map[string]string - - // Map of request URL to static files generated - StaticFiles map[string]string - - // Links to all markdown pages ordered by date. - Links []PageLink - - // Stores a directory hash to check for changes. - linksHash string - - // Directory to store static files - StaticDir string - - // If in development mode. i.e. Actively editing markdown files. - Development bool - - sync.RWMutex -} - -// IsValidExt checks to see if an extension is a valid markdown extension -// for config. -func (c *Config) IsValidExt(ext string) bool { - for _, e := range c.Extensions { - if e == ext { - return true - } - } - return false } // ServeHTTP implements the http.Handler interface. @@ -104,69 +75,39 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error fpath = idx } - for _, ext := range cfg.Extensions { - if strings.HasSuffix(fpath, ext) { - f, err := md.FileSys.Open(fpath) - if err != nil { - if os.IsPermission(err) { - return http.StatusForbidden, err - } - return http.StatusNotFound, nil + // If supported extension, process it + if _, ok := cfg.Extensions[path.Ext(fpath)]; ok { + f, err := md.FileSys.Open(fpath) + if err != nil { + if os.IsPermission(err) { + return http.StatusForbidden, err } - - fs, err := f.Stat() - if err != nil { - return http.StatusNotFound, nil - } - - // if development is set, scan directory for file changes for links. - if cfg.Development { - if err := GenerateStatic(md, cfg); err != nil { - log.Printf("[ERROR] markdown: on-demand site generation error: %v", err) - } - } - - cfg.RLock() - filepath, ok := cfg.StaticFiles[fpath] - cfg.RUnlock() - // if static site is generated, attempt to use it - if ok { - if fs1, err := os.Stat(filepath); err == nil { - // if markdown has not been modified since static page - // generation, serve the static page - if fs.ModTime().Before(fs1.ModTime()) { - if html, err := ioutil.ReadFile(filepath); err == nil { - middleware.SetLastModifiedHeader(w, fs1.ModTime()) - w.Write(html) - return http.StatusOK, nil - } - if os.IsPermission(err) { - return http.StatusForbidden, err - } - return http.StatusNotFound, nil - } - } - } - - body, err := ioutil.ReadAll(f) - if err != nil { - return http.StatusInternalServerError, err - } - - ctx := middleware.Context{ - Root: md.FileSys, - Req: r, - URL: r.URL, - } - html, err := md.Process(cfg, fpath, body, ctx) - if err != nil { - return http.StatusInternalServerError, err - } - - middleware.SetLastModifiedHeader(w, fs.ModTime()) - w.Write(html) - return http.StatusOK, nil + return http.StatusNotFound, nil } + + fs, err := f.Stat() + if err != nil { + return http.StatusNotFound, nil + } + + body, err := ioutil.ReadAll(f) + if err != nil { + return http.StatusInternalServerError, err + } + + ctx := middleware.Context{ + Root: md.FileSys, + Req: r, + URL: r.URL, + } + html, err := md.Process(cfg, fpath, body, ctx) + if err != nil { + return http.StatusInternalServerError, err + } + + middleware.SetLastModifiedHeader(w, fs.ModTime()) + w.Write(html) + return http.StatusOK, nil } } diff --git a/middleware/markdown/markdown_test.go b/middleware/markdown/markdown_test.go index e8796d11..fbfb7379 100644 --- a/middleware/markdown/markdown_test.go +++ b/middleware/markdown/markdown_test.go @@ -2,12 +2,10 @@ package markdown import ( "bufio" - "log" "net/http" "net/http/httptest" "os" "strings" - "sync" "testing" "time" @@ -23,54 +21,46 @@ func TestMarkdown(t *testing.T) { FileSys: http.Dir("./testdata"), Configs: []*Config{ { - Renderer: blackfriday.HtmlRenderer(0, "", ""), - PathScope: "/blog", - Extensions: []string{".md"}, - Styles: []string{}, - Scripts: []string{}, - Templates: templates, - StaticDir: DefaultStaticDir, - StaticFiles: make(map[string]string), + Renderer: blackfriday.HtmlRenderer(0, "", ""), + PathScope: "/blog", + Extensions: map[string]struct{}{ + ".md": struct{}{}, + }, + Styles: []string{}, + Scripts: []string{}, + Templates: templates, }, { - Renderer: blackfriday.HtmlRenderer(0, "", ""), - PathScope: "/docflags", - Extensions: []string{".md"}, - Styles: []string{}, - Scripts: []string{}, + Renderer: blackfriday.HtmlRenderer(0, "", ""), + PathScope: "/docflags", + Extensions: map[string]struct{}{ + ".md": struct{}{}, + }, + Styles: []string{}, + Scripts: []string{}, Templates: map[string]string{ DefaultTemplate: "testdata/docflags/template.txt", }, - StaticDir: DefaultStaticDir, - StaticFiles: make(map[string]string), }, { - Renderer: blackfriday.HtmlRenderer(0, "", ""), - PathScope: "/log", - Extensions: []string{".md"}, - Styles: []string{"/resources/css/log.css", "/resources/css/default.css"}, - Scripts: []string{"/resources/js/log.js", "/resources/js/default.js"}, - Templates: make(map[string]string), - StaticDir: DefaultStaticDir, - StaticFiles: make(map[string]string), - }, - { - Renderer: blackfriday.HtmlRenderer(0, "", ""), - PathScope: "/og", - Extensions: []string{".md"}, - Styles: []string{}, - Scripts: []string{}, - Templates: templates, - StaticDir: "testdata/og_static", - StaticFiles: map[string]string{"/og/first.md": "testdata/og_static/og/first.md/index.html"}, - Links: []PageLink{ - { - Title: "first", - Summary: "", - Date: time.Now(), - URL: "/og/first.md", - }, + Renderer: blackfriday.HtmlRenderer(0, "", ""), + PathScope: "/log", + Extensions: map[string]struct{}{ + ".md": struct{}{}, }, + Styles: []string{"/resources/css/log.css", "/resources/css/default.css"}, + Scripts: []string{"/resources/js/log.js", "/resources/js/default.js"}, + Templates: make(map[string]string), + }, + { + Renderer: blackfriday.HtmlRenderer(0, "", ""), + PathScope: "/og", + Extensions: map[string]struct{}{ + ".md": struct{}{}, + }, + Styles: []string{}, + Scripts: []string{}, + Templates: templates, }, }, IndexFiles: []string{"index.html"}, @@ -80,14 +70,6 @@ func TestMarkdown(t *testing.T) { }), } - for i := range md.Configs { - c := md.Configs[i] - if err := GenerateStatic(md, c); err != nil { - t.Fatalf("Error: %v", err) - } - Watch(md, c, time.Millisecond*100) - } - req, err := http.NewRequest("GET", "/blog/test.md", nil) if err != nil { t.Fatalf("Could not create HTTP request: %v", err) @@ -219,52 +201,6 @@ Welcome to title! if !equalStrings(respBody, expectedBody) { t.Fatalf("Expected body: %v got: %v", expectedBody, respBody) } - - expectedLinks := []string{ - "/blog/test.md", - "/docflags/test.md", - "/log/test.md", - } - - for i, c := range md.Configs[:2] { - log.Printf("Test number: %d, configuration links: %v, config: %v", i, c.Links, c) - if c.Links[0].URL != expectedLinks[i] { - t.Fatalf("Expected %v got %v", expectedLinks[i], c.Links[0].URL) - } - } - - // attempt to trigger race conditions - var w sync.WaitGroup - f := func() { - req, err := http.NewRequest("GET", "/log/test.md", nil) - if err != nil { - t.Fatalf("Could not create HTTP request: %v", err) - } - rec := httptest.NewRecorder() - - md.ServeHTTP(rec, req) - w.Done() - } - for i := 0; i < 5; i++ { - w.Add(1) - go f() - } - w.Wait() - - f = func() { - GenerateStatic(md, md.Configs[0]) - w.Done() - } - for i := 0; i < 5; i++ { - w.Add(1) - go f() - } - w.Wait() - - if err = os.RemoveAll(DefaultStaticDir); err != nil { - t.Errorf("Error while removing the generated static files: %v", err) - } - } func equalStrings(s1, s2 string) bool { diff --git a/middleware/markdown/metadata.go b/middleware/markdown/metadata.go index 9b5c416a..20a30e18 100644 --- a/middleware/markdown/metadata.go +++ b/middleware/markdown/metadata.go @@ -2,12 +2,16 @@ package markdown import ( "bytes" - "encoding/json" "fmt" "time" +) - "github.com/BurntSushi/toml" - "gopkg.in/yaml.v2" +var ( + // Date format YYYY-MM-DD HH:MM:SS or YYYY-MM-DD + timeLayout = []string{ + `2006-01-02 15:04:05`, + `2006-01-02`, + } ) // Metadata stores a page's metadata @@ -30,6 +34,8 @@ type Metadata struct { // load loads parsed values in parsedMap into Metadata func (m *Metadata) load(parsedMap map[string]interface{}) { + + // Pull top level things out if title, ok := parsedMap["title"]; ok { m.Title, _ = title.(string) } @@ -37,17 +43,21 @@ func (m *Metadata) load(parsedMap map[string]interface{}) { m.Template, _ = template.(string) } if date, ok := parsedMap["date"].(string); ok { - if t, err := time.Parse(timeLayout, date); err == nil { - m.Date = t + for _, layout := range timeLayout { + if t, err := time.Parse(layout, date); err == nil { + m.Date = t + break + } } } - // store everything as a variable + + // Store everything as a flag or variable for key, val := range parsedMap { switch v := val.(type) { - case string: - m.Variables[key] = v case bool: m.Flags[key] = v + case string: + m.Variables[key] = v } } } @@ -70,116 +80,6 @@ type MetadataParser interface { Metadata() Metadata } -// JSONMetadataParser is the MetadataParser for JSON -type JSONMetadataParser struct { - metadata Metadata -} - -// Parse the metadata -func (j *JSONMetadataParser) Parse(b []byte) ([]byte, error) { - b, markdown, err := extractMetadata(j, b) - if err != nil { - return markdown, err - } - m := make(map[string]interface{}) - - // Read the preceding JSON object - decoder := json.NewDecoder(bytes.NewReader(b)) - if err := decoder.Decode(&m); err != nil { - return markdown, err - } - j.metadata.load(m) - - return markdown, nil -} - -// Metadata returns parsed metadata. It should be called -// only after a call to Parse returns without error. -func (j *JSONMetadataParser) Metadata() Metadata { - return j.metadata -} - -// Opening returns the opening identifier JSON metadata -func (j *JSONMetadataParser) Opening() []byte { - return []byte("{") -} - -// Closing returns the closing identifier JSON metadata -func (j *JSONMetadataParser) Closing() []byte { - return []byte("}") -} - -// TOMLMetadataParser is the MetadataParser for TOML -type TOMLMetadataParser struct { - metadata Metadata -} - -// Parse the metadata -func (t *TOMLMetadataParser) Parse(b []byte) ([]byte, error) { - b, markdown, err := extractMetadata(t, b) - if err != nil { - return markdown, err - } - m := make(map[string]interface{}) - if err := toml.Unmarshal(b, &m); err != nil { - return markdown, err - } - t.metadata.load(m) - return markdown, nil -} - -// Metadata returns parsed metadata. It should be called -// only after a call to Parse returns without error. -func (t *TOMLMetadataParser) Metadata() Metadata { - return t.metadata -} - -// Opening returns the opening identifier TOML metadata -func (t *TOMLMetadataParser) Opening() []byte { - return []byte("+++") -} - -// Closing returns the closing identifier TOML metadata -func (t *TOMLMetadataParser) Closing() []byte { - return []byte("+++") -} - -// YAMLMetadataParser is the MetadataParser for YAML -type YAMLMetadataParser struct { - metadata Metadata -} - -// Parse the metadata -func (y *YAMLMetadataParser) Parse(b []byte) ([]byte, error) { - b, markdown, err := extractMetadata(y, b) - if err != nil { - return markdown, err - } - - m := make(map[string]interface{}) - if err := yaml.Unmarshal(b, &m); err != nil { - return markdown, err - } - y.metadata.load(m) - return markdown, nil -} - -// Metadata returns parsed metadata. It should be called -// only after a call to Parse returns without error. -func (y *YAMLMetadataParser) Metadata() Metadata { - return y.metadata -} - -// Opening returns the opening identifier YAML metadata -func (y *YAMLMetadataParser) Opening() []byte { - return []byte("---") -} - -// Closing returns the closing identifier YAML metadata -func (y *YAMLMetadataParser) Closing() []byte { - return []byte("---") -} - // extractMetadata separates metadata content from from markdown content in b. // It returns the metadata, the remaining bytes (markdown), and an error, if any. func extractMetadata(parser MetadataParser, b []byte) (metadata []byte, markdown []byte, err error) { diff --git a/middleware/markdown/metadata_json.go b/middleware/markdown/metadata_json.go new file mode 100644 index 00000000..53dbea19 --- /dev/null +++ b/middleware/markdown/metadata_json.go @@ -0,0 +1,45 @@ +package markdown + +import ( + "bytes" + "encoding/json" +) + +// JSONMetadataParser is the MetadataParser for JSON +type JSONMetadataParser struct { + metadata Metadata +} + +// Parse the metadata +func (j *JSONMetadataParser) Parse(b []byte) ([]byte, error) { + b, markdown, err := extractMetadata(j, b) + if err != nil { + return markdown, err + } + m := make(map[string]interface{}) + + // Read the preceding JSON object + decoder := json.NewDecoder(bytes.NewReader(b)) + if err := decoder.Decode(&m); err != nil { + return markdown, err + } + j.metadata.load(m) + + return markdown, nil +} + +// Metadata returns parsed metadata. It should be called +// only after a call to Parse returns without error. +func (j *JSONMetadataParser) Metadata() Metadata { + return j.metadata +} + +// Opening returns the opening identifier JSON metadata +func (j *JSONMetadataParser) Opening() []byte { + return []byte("{") +} + +// Closing returns the closing identifier JSON metadata +func (j *JSONMetadataParser) Closing() []byte { + return []byte("}") +} diff --git a/middleware/markdown/metadata_toml.go b/middleware/markdown/metadata_toml.go new file mode 100644 index 00000000..fe4068d7 --- /dev/null +++ b/middleware/markdown/metadata_toml.go @@ -0,0 +1,40 @@ +package markdown + +import ( + "github.com/BurntSushi/toml" +) + +// TOMLMetadataParser is the MetadataParser for TOML +type TOMLMetadataParser struct { + metadata Metadata +} + +// Parse the metadata +func (t *TOMLMetadataParser) Parse(b []byte) ([]byte, error) { + b, markdown, err := extractMetadata(t, b) + if err != nil { + return markdown, err + } + m := make(map[string]interface{}) + if err := toml.Unmarshal(b, &m); err != nil { + return markdown, err + } + t.metadata.load(m) + return markdown, nil +} + +// Metadata returns parsed metadata. It should be called +// only after a call to Parse returns without error. +func (t *TOMLMetadataParser) Metadata() Metadata { + return t.metadata +} + +// Opening returns the opening identifier TOML metadata +func (t *TOMLMetadataParser) Opening() []byte { + return []byte("+++") +} + +// Closing returns the closing identifier TOML metadata +func (t *TOMLMetadataParser) Closing() []byte { + return []byte("+++") +} diff --git a/middleware/markdown/metadata_yaml.go b/middleware/markdown/metadata_yaml.go new file mode 100644 index 00000000..41103e21 --- /dev/null +++ b/middleware/markdown/metadata_yaml.go @@ -0,0 +1,41 @@ +package markdown + +import ( + "gopkg.in/yaml.v2" +) + +// YAMLMetadataParser is the MetadataParser for YAML +type YAMLMetadataParser struct { + metadata Metadata +} + +// Parse the metadata +func (y *YAMLMetadataParser) Parse(b []byte) ([]byte, error) { + b, markdown, err := extractMetadata(y, b) + if err != nil { + return markdown, err + } + + m := make(map[string]interface{}) + if err := yaml.Unmarshal(b, &m); err != nil { + return markdown, err + } + y.metadata.load(m) + return markdown, nil +} + +// Metadata returns parsed metadata. It should be called +// only after a call to Parse returns without error. +func (y *YAMLMetadataParser) Metadata() Metadata { + return y.metadata +} + +// Opening returns the opening identifier YAML metadata +func (y *YAMLMetadataParser) Opening() []byte { + return []byte("---") +} + +// Closing returns the closing identifier YAML metadata +func (y *YAMLMetadataParser) Closing() []byte { + return []byte("---") +} diff --git a/middleware/markdown/page.go b/middleware/markdown/page.go deleted file mode 100644 index 9266d9c4..00000000 --- a/middleware/markdown/page.go +++ /dev/null @@ -1,169 +0,0 @@ -package markdown - -import ( - "bytes" - "io/ioutil" - "log" - "os" - "path/filepath" - "sort" - "strings" - "sync" - "time" - - "github.com/russross/blackfriday" -) - -const ( - // Date format YYYY-MM-DD HH:MM:SS - timeLayout = `2006-01-02 15:04:05` - - // Maximum length of page summary. - summaryLen = 500 -) - -// PageLink represents a statically generated markdown page. -type PageLink struct { - Title string - Summary string - Date time.Time - URL string -} - -// byDate sorts PageLink by newest date to oldest. -type byDate []PageLink - -func (p byDate) Len() int { return len(p) } -func (p byDate) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p byDate) Less(i, j int) bool { return p[i].Date.After(p[j].Date) } - -type linkGen struct { - generating bool - waiters int - lastErr error - sync.RWMutex - sync.WaitGroup -} - -func (l *linkGen) addWaiter() { - l.WaitGroup.Add(1) - l.waiters++ -} - -func (l *linkGen) discardWaiters() { - l.Lock() - defer l.Unlock() - for i := 0; i < l.waiters; i++ { - l.Done() - } -} - -func (l *linkGen) started() bool { - l.RLock() - defer l.RUnlock() - return l.generating -} - -// generateLinks generate links to markdown files if there are file changes. -// It returns true when generation is done and false otherwise. -func (l *linkGen) generateLinks(md Markdown, cfg *Config) bool { - l.Lock() - l.generating = true - l.Unlock() - - fp := filepath.Join(md.Root, cfg.PathScope) // path to scan for .md files - - // If the file path to scan for Markdown files (fp) does - // not exist, there are no markdown files to scan for. - if _, err := os.Stat(fp); os.IsNotExist(err) { - l.Lock() - l.lastErr = err - l.generating = false - l.Unlock() - return false - } - - hash, err := computeDirHash(md, cfg) - - // same hash, return. - if err == nil && hash == cfg.linksHash { - l.Lock() - l.generating = false - l.Unlock() - return false - } else if err != nil { - log.Printf("[ERROR] markdown: Hash error: %v", err) - } - - cfg.Links = []PageLink{} - - cfg.Lock() - l.lastErr = filepath.Walk(fp, func(path string, info os.FileInfo, err error) error { - for _, ext := range cfg.Extensions { - if info.IsDir() || !strings.HasSuffix(info.Name(), ext) { - continue - } - - // Load the file - body, err := ioutil.ReadFile(path) - if err != nil { - return err - } - - // Get the relative path as if it were a HTTP request, - // then prepend with "/" (like a real HTTP request) - reqPath, err := filepath.Rel(md.Root, path) - if err != nil { - return err - } - reqPath = "/" + filepath.ToSlash(reqPath) - - // Make the summary - parser := findParser(body) - if parser == nil { - // no metadata, ignore. - continue - } - summaryRaw, err := parser.Parse(body) - if err != nil { - return err - } - summary := blackfriday.Markdown(summaryRaw, SummaryRenderer{}, 0) - - // truncate summary to maximum length - if len(summary) > summaryLen { - summary = summary[:summaryLen] - - // trim to nearest word - lastSpace := bytes.LastIndex(summary, []byte(" ")) - if lastSpace != -1 { - summary = summary[:lastSpace] - } - } - - metadata := parser.Metadata() - - cfg.Links = append(cfg.Links, PageLink{ - Title: metadata.Title, - URL: reqPath, - Date: metadata.Date, - Summary: string(summary), - }) - - break // don't try other file extensions - } - - return nil - }) - - // sort by newest date - sort.Sort(byDate(cfg.Links)) - - cfg.linksHash = hash - cfg.Unlock() - - l.Lock() - l.generating = false - l.Unlock() - return true -} diff --git a/middleware/markdown/process.go b/middleware/markdown/process.go index 94887d0e..e9768627 100644 --- a/middleware/markdown/process.go +++ b/middleware/markdown/process.go @@ -3,10 +3,7 @@ package markdown import ( "bytes" "io/ioutil" - "log" - "os" "path/filepath" - "strings" "text/template" "github.com/mholt/caddy/middleware" @@ -16,8 +13,6 @@ import ( const ( // DefaultTemplate is the default template. DefaultTemplate = "defaultTemplate" - // DefaultStaticDir is the default static directory. - DefaultStaticDir = "generated_site" ) // Data represents a markdown document. @@ -25,7 +20,8 @@ type Data struct { middleware.Context Doc map[string]string DocFlags map[string]bool - Links []PageLink + Styles []string + Scripts []string } // Include "overrides" the embedded middleware.Context's Include() @@ -75,7 +71,11 @@ func (md Markdown) Process(c *Config, requestPath string, b []byte, ctx middlewa } // process markdown - extns := blackfriday.EXTENSION_TABLES | blackfriday.EXTENSION_FENCED_CODE | blackfriday.EXTENSION_STRIKETHROUGH | blackfriday.EXTENSION_DEFINITION_LISTS + extns := 0 + extns |= blackfriday.EXTENSION_TABLES + extns |= blackfriday.EXTENSION_FENCED_CODE + extns |= blackfriday.EXTENSION_STRIKETHROUGH + extns |= blackfriday.EXTENSION_DEFINITION_LISTS markdown = blackfriday.Markdown(markdown, c.Renderer, extns) // set it as body for template @@ -94,123 +94,51 @@ func (md Markdown) Process(c *Config, requestPath string, b []byte, ctx middlewa // processTemplate processes a template given a requestPath, // template (tmpl) and metadata func (md Markdown) processTemplate(c *Config, requestPath string, tmpl []byte, metadata Metadata, ctx middleware.Context) ([]byte, error) { + var t *template.Template + var err error + // if template is not specified, // use the default template if tmpl == nil { - tmpl = defaultTemplate(c, metadata, requestPath) + t = template.Must(template.New("").Parse(htmlTemplate)) + } else { + t, err = template.New("").Parse(string(tmpl)) + if err != nil { + return nil, err + } } // process the template - b := new(bytes.Buffer) - t, err := template.New("").Parse(string(tmpl)) - if err != nil { - return nil, err - } mdData := Data{ Context: ctx, Doc: metadata.Variables, DocFlags: metadata.Flags, - Links: c.Links, + Styles: c.Styles, + Scripts: c.Scripts, } - c.RLock() + b := new(bytes.Buffer) err = t.Execute(b, mdData) - c.RUnlock() - if err != nil { return nil, err } - // generate static page - if err = md.generatePage(c, requestPath, b.Bytes()); err != nil { - // if static page generation fails, nothing fatal, only log the error. - // TODO: Report (return) this non-fatal error, but don't log it here? - log.Println("[ERROR] markdown: Render:", err) - } - return b.Bytes(), nil - -} - -// generatePage generates a static html page from the markdown in content if c.StaticDir -// is a non-empty value, meaning that the user enabled static site generation. -func (md Markdown) generatePage(c *Config, requestPath string, content []byte) error { - // Only generate the page if static site generation is enabled - if c.StaticDir != "" { - // if static directory is not existing, create it - if _, err := os.Stat(c.StaticDir); err != nil { - err := os.MkdirAll(c.StaticDir, os.FileMode(0755)) - if err != nil { - return err - } - } - - // the URL will always use "/" as a path separator, - // convert that to a native path to support OS that - // use different path separators - filePath := filepath.Join(c.StaticDir, filepath.FromSlash(requestPath)) - - // If it is index file, use the directory instead - if md.IsIndexFile(filepath.Base(requestPath)) { - filePath, _ = filepath.Split(filePath) - } - - // Create the directory in case it is not existing - if err := os.MkdirAll(filePath, os.FileMode(0744)); err != nil { - return err - } - - // generate index.html file in the directory - filePath = filepath.Join(filePath, "index.html") - err := ioutil.WriteFile(filePath, content, os.FileMode(0664)) - if err != nil { - return err - } - - c.Lock() - c.StaticFiles[requestPath] = filepath.ToSlash(filePath) - c.Unlock() - } - - return nil -} - -// defaultTemplate constructs a default template. -func defaultTemplate(c *Config, metadata Metadata, requestPath string) []byte { - var scripts, styles bytes.Buffer - for _, style := range c.Styles { - styles.WriteString(strings.Replace(cssTemplate, "{{url}}", style, 1)) - styles.WriteString("\r\n") - } - for _, script := range c.Scripts { - scripts.WriteString(strings.Replace(jsTemplate, "{{url}}", script, 1)) - scripts.WriteString("\r\n") - } - - // Title is first line (length-limited), otherwise filename - title, _ := metadata.Variables["title"] - - html := []byte(htmlTemplate) - html = bytes.Replace(html, []byte("{{title}}"), []byte(title), 1) - html = bytes.Replace(html, []byte("{{css}}"), styles.Bytes(), 1) - html = bytes.Replace(html, []byte("{{js}}"), scripts.Bytes(), 1) - - return html } const ( htmlTemplate = ` - {{title}} + {{.Doc.title}} - {{css}} - {{js}} + {{range .Styles}} + {{end -}} + {{range .Scripts}} + {{end -}} {{.Doc.body}} ` - cssTemplate = `` - jsTemplate = `` ) diff --git a/middleware/markdown/renderer.go b/middleware/markdown/renderer.go deleted file mode 100644 index 44c0163d..00000000 --- a/middleware/markdown/renderer.go +++ /dev/null @@ -1,139 +0,0 @@ -package markdown - -import ( - "bytes" -) - -// SummaryRenderer represents a summary renderer. -type SummaryRenderer struct{} - -// Block-level callbacks - -// BlockCode is the code tag callback. -func (r SummaryRenderer) BlockCode(out *bytes.Buffer, text []byte, lang string) {} - -// BlockQuote is the quote tag callback. -func (r SummaryRenderer) BlockQuote(out *bytes.Buffer, text []byte) {} - -// BlockHtml is the HTML tag callback. -func (r SummaryRenderer) BlockHtml(out *bytes.Buffer, text []byte) {} - -// Header is the header tag callback. -func (r SummaryRenderer) Header(out *bytes.Buffer, text func() bool, level int, id string) {} - -// HRule is the horizontal rule tag callback. -func (r SummaryRenderer) HRule(out *bytes.Buffer) {} - -// List is the list tag callback. -func (r SummaryRenderer) List(out *bytes.Buffer, text func() bool, flags int) { - // TODO: This is not desired (we'd rather not write lists as part of summary), - // but see this issue: https://github.com/russross/blackfriday/issues/189 - marker := out.Len() - if !text() { - out.Truncate(marker) - } - out.Write([]byte{' '}) -} - -// ListItem is the list item tag callback. -func (r SummaryRenderer) ListItem(out *bytes.Buffer, text []byte, flags int) {} - -// Paragraph is the paragraph tag callback. -func (r SummaryRenderer) Paragraph(out *bytes.Buffer, text func() bool) { - marker := out.Len() - if !text() { - out.Truncate(marker) - } - out.Write([]byte{' '}) -} - -// Table is the table tag callback. -func (r SummaryRenderer) Table(out *bytes.Buffer, header []byte, body []byte, columnData []int) {} - -// TableRow is the table row tag callback. -func (r SummaryRenderer) TableRow(out *bytes.Buffer, text []byte) {} - -// TableHeaderCell is the table header cell tag callback. -func (r SummaryRenderer) TableHeaderCell(out *bytes.Buffer, text []byte, flags int) {} - -// TableCell is the table cell tag callback. -func (r SummaryRenderer) TableCell(out *bytes.Buffer, text []byte, flags int) {} - -// Footnotes is the foot notes tag callback. -func (r SummaryRenderer) Footnotes(out *bytes.Buffer, text func() bool) {} - -// FootnoteItem is the footnote item tag callback. -func (r SummaryRenderer) FootnoteItem(out *bytes.Buffer, name, text []byte, flags int) {} - -// TitleBlock is the title tag callback. -func (r SummaryRenderer) TitleBlock(out *bytes.Buffer, text []byte) {} - -// Span-level callbacks - -// AutoLink is the autolink tag callback. -func (r SummaryRenderer) AutoLink(out *bytes.Buffer, link []byte, kind int) {} - -// CodeSpan is the code span tag callback. -func (r SummaryRenderer) CodeSpan(out *bytes.Buffer, text []byte) { - out.Write([]byte("`")) - out.Write(text) - out.Write([]byte("`")) -} - -// DoubleEmphasis is the double emphasis tag callback. -func (r SummaryRenderer) DoubleEmphasis(out *bytes.Buffer, text []byte) { - out.Write(text) -} - -// Emphasis is the emphasis tag callback. -func (r SummaryRenderer) Emphasis(out *bytes.Buffer, text []byte) { - out.Write(text) -} - -// Image is the image tag callback. -func (r SummaryRenderer) Image(out *bytes.Buffer, link []byte, title []byte, alt []byte) {} - -// LineBreak is the line break tag callback. -func (r SummaryRenderer) LineBreak(out *bytes.Buffer) {} - -// Link is the link tag callback. -func (r SummaryRenderer) Link(out *bytes.Buffer, link []byte, title []byte, content []byte) { - out.Write(content) -} - -// RawHtmlTag is the raw HTML tag callback. -func (r SummaryRenderer) RawHtmlTag(out *bytes.Buffer, tag []byte) {} - -// TripleEmphasis is the triple emphasis tag callback. -func (r SummaryRenderer) TripleEmphasis(out *bytes.Buffer, text []byte) { - out.Write(text) -} - -// StrikeThrough is the strikethrough tag callback. -func (r SummaryRenderer) StrikeThrough(out *bytes.Buffer, text []byte) {} - -// FootnoteRef is the footnote ref tag callback. -func (r SummaryRenderer) FootnoteRef(out *bytes.Buffer, ref []byte, id int) {} - -// Low-level callbacks - -// Entity callback. -func (r SummaryRenderer) Entity(out *bytes.Buffer, entity []byte) { - out.Write(entity) -} - -// NormalText callback. -func (r SummaryRenderer) NormalText(out *bytes.Buffer, text []byte) { - out.Write(text) -} - -// Header and footer - -// DocumentHeader callback. -func (r SummaryRenderer) DocumentHeader(out *bytes.Buffer) {} - -// DocumentFooter callback. -func (r SummaryRenderer) DocumentFooter(out *bytes.Buffer) {} - -// GetFlags returns zero. -func (r SummaryRenderer) GetFlags() int { return 0 } diff --git a/middleware/markdown/watcher.go b/middleware/markdown/watcher.go deleted file mode 100644 index fa5fb3ee..00000000 --- a/middleware/markdown/watcher.go +++ /dev/null @@ -1,42 +0,0 @@ -package markdown - -import ( - "log" - "time" -) - -// DefaultInterval is the default interval at which the markdown watcher -// checks for changes. -const DefaultInterval = time.Second * 60 - -// Watch monitors the configured markdown directory for changes. It calls GenerateLinks -// when there are changes. -func Watch(md Markdown, c *Config, interval time.Duration) (stopChan chan struct{}) { - return TickerFunc(interval, func() { - if err := GenerateStatic(md, c); err != nil { - log.Printf("[ERROR] markdown: Re-generating static site: %v", err) - } - }) -} - -// TickerFunc runs f at interval. A message to the returned channel will stop the -// executing goroutine. -func TickerFunc(interval time.Duration, f func()) chan struct{} { - stopChan := make(chan struct{}) - - ticker := time.NewTicker(interval) - go func() { - loop: - for { - select { - case <-ticker.C: - f() - case <-stopChan: - ticker.Stop() - break loop - } - } - }() - - return stopChan -} diff --git a/middleware/markdown/watcher_test.go b/middleware/markdown/watcher_test.go deleted file mode 100644 index 8a89e007..00000000 --- a/middleware/markdown/watcher_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package markdown - -import ( - "fmt" - "strings" - "sync" - "testing" - "time" -) - -func TestWatcher(t *testing.T) { - expected := "12345678" - interval := time.Millisecond * 100 - i := 0 - out := "" - syncChan := make(chan struct{}) - stopChan := TickerFunc(interval, func() { - i++ - out += fmt.Sprint(i) - syncChan <- struct{}{} - }) - sleepInSync(8, syncChan, stopChan) - if out != expected { - t.Fatalf("Expected to have prefix %v, found %v", expected, out) - } - out = "" - i = 0 - var mu sync.Mutex - stopChan = TickerFunc(interval, func() { - i++ - mu.Lock() - out += fmt.Sprint(i) - mu.Unlock() - syncChan <- struct{}{} - }) - sleepInSync(9, syncChan, stopChan) - mu.Lock() - res := out - mu.Unlock() - if !strings.HasPrefix(res, expected) || res == expected { - t.Fatalf("expected (%v) must be a proper prefix of out(%v).", expected, out) - } -} - -func sleepInSync(times int, syncChan chan struct{}, stopChan chan struct{}) { - for i := 0; i < times; i++ { - <-syncChan - } - stopChan <- struct{}{} -}