Merge pull request #68 from abiosoft/master

Markdown: support for templates and metadata

config/setup/markdown.go

@@ -2,6 +2,8 @@ package setup
 
 import (
 	"net/http"
+	"path"
+	"path/filepath"
 
 	"github.com/mholt/caddy/middleware"
 	"github.com/mholt/caddy/middleware/markdown"
@@ -34,6 +36,9 @@ func markdownParse(c *Controller) ([]markdown.Config, error) {
 	for c.Next() {
 		md := markdown.Config{
 			Renderer:    blackfriday.HtmlRenderer(0, "", ""),
+			Templates:   make(map[string]string),
+			StaticFiles: make(map[string]string),
+			StaticDir:   path.Join(c.Root, markdown.StaticDir),
 		}
 
 		// Get the path scope
@@ -61,6 +66,23 @@ func markdownParse(c *Controller) ([]markdown.Config, error) {
 					return mdconfigs, c.ArgErr()
 				}
 				md.Scripts = append(md.Scripts, c.Val())
+			case "template":
+				tArgs := c.RemainingArgs()
+				switch len(tArgs) {
+				case 0:
+					return mdconfigs, c.ArgErr()
+				case 1:
+					if _, ok := md.Templates[markdown.DefaultTemplate]; ok {
+						return mdconfigs, c.Err("only one default template is allowed, use alias.")
+					}
+					fpath := filepath.Clean(c.Root + string(filepath.Separator) + tArgs[0])
+					md.Templates[markdown.DefaultTemplate] = fpath
+				case 2:
+					fpath := filepath.Clean(c.Root + string(filepath.Separator) + tArgs[1])
+					md.Templates[tArgs[0]] = fpath
+				default:
+					return mdconfigs, c.ArgErr()
+				}
 			default:
 				return mdconfigs, c.Err("Expected valid markdown configuration property")
 			}

middleware/markdown/markdown.go

@@ -3,11 +3,9 @@
 package markdown
 
 import (
-	"bytes"
 	"io/ioutil"
 	"net/http"
 	"os"
-	"path"
 	"strings"
 
 	"github.com/mholt/caddy/middleware"
@@ -33,6 +31,16 @@ type Markdown struct {
 	IndexFiles []string
 }
 
+// Helper function to check if a file is an index file
+func (m Markdown) IsIndexFile(file string) bool {
+	for _, f := range m.IndexFiles {
+		if f == file {
+			return true
+		}
+	}
+	return false
+}
+
 // Config stores markdown middleware configurations.
 type Config struct {
 	// Markdown renderer
@@ -49,6 +57,15 @@ type Config struct {
 
 	// List of JavaScript files to load for each markdown file
 	Scripts []string
+
+	// Map of registered templates
+	Templates map[string]string
+
+	// Map of request URL to static files generated
+	StaticFiles map[string]string
+
+	// Directory to store static files
+	StaticDir string
 }
 
 // ServeHTTP implements the http.Handler interface.
@@ -73,44 +90,37 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error
 					return http.StatusNotFound, nil
 				}
 
+				fs, err := f.Stat()
+				if err != nil {
+					return http.StatusNotFound, nil
+				}
+
+				// if static site is generated, attempt to use it
+				if filepath, ok := m.StaticFiles[fpath]; ok {
+					if fs1, err := os.Stat(filepath); err == nil {
+						// if markdown has not been modified
+						// since static page generation,
+						// serve the static page
+						if fs.ModTime().UnixNano() < fs1.ModTime().UnixNano() {
+							if html, err := ioutil.ReadFile(filepath); err == nil {
+								w.Write(html)
+								return http.StatusOK, nil
+							}
+						}
+					}
+				}
+
 				body, err := ioutil.ReadAll(f)
 				if err != nil {
 					return http.StatusInternalServerError, err
 				}
 
-				content := blackfriday.Markdown(body, m.Renderer, 0)
-
-				var scripts, styles string
-				for _, style := range m.Styles {
-					styles += strings.Replace(cssTemplate, "{{url}}", style, 1) + "\r\n"
-				}
-				for _, script := range m.Scripts {
-					scripts += strings.Replace(jsTemplate, "{{url}}", script, 1) + "\r\n"
+				html, err := md.process(m, fpath, body)
+				if err != nil {
+					return http.StatusInternalServerError, err
 				}
 
-				// Title is first line (length-limited), otherwise filename
-				title := path.Base(fpath)
-				newline := bytes.Index(body, []byte("\n"))
-				if newline > -1 {
-					firstline := body[:newline]
-					newTitle := strings.TrimSpace(string(firstline))
-					if len(newTitle) > 1 {
-						if len(newTitle) > 128 {
-							title = newTitle[:128]
-						} else {
-							title = newTitle
-						}
-					}
-				}
-
-				html := htmlTemplate
-				html = strings.Replace(html, "{{title}}", title, 1)
-				html = strings.Replace(html, "{{css}}", styles, 1)
-				html = strings.Replace(html, "{{js}}", scripts, 1)
-				html = strings.Replace(html, "{{body}}", string(content), 1)
-
-				w.Write([]byte(html))
-
+				w.Write(html)
 				return http.StatusOK, nil
 			}
 		}
@@ -119,20 +129,3 @@ func (md Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error
 	// Didn't qualify to serve as markdown; pass-thru
 	return md.Next.ServeHTTP(w, r)
 }
-
-const (
-	htmlTemplate = `<!DOCTYPE html>
-<html>
-	<head>
-		<title>{{title}}</title>
-		<meta charset="utf-8">
-		{{css}}
-		{{js}}
-	</head>
-	<body>
-		{{body}}
-	</body>
-</html>`
-	cssTemplate = `<link rel="stylesheet" href="{{url}}">`
-	jsTemplate  = `<script src="{{url}}"></script>`
-)

middleware/markdown/metadata.go

@@ -0,0 +1,239 @@
+package markdown
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+
+	"github.com/BurntSushi/toml"
+	"gopkg.in/yaml.v2"
+)
+
+var (
+	parsers = []MetadataParser{
+		&JSONMetadataParser{},
+		&TOMLMetadataParser{},
+		&YAMLMetadataParser{},
+	}
+)
+
+// Metadata stores a page's metadata
+type Metadata struct {
+	// Page title
+	Title string
+
+	// Page template
+	Template string
+
+	// Variables to be used with Template
+	Variables map[string]interface{}
+}
+
+// load loads parsed values in parsedMap into Metadata
+func (m *Metadata) load(parsedMap map[string]interface{}) {
+	if template, ok := parsedMap["title"]; ok {
+		m.Title, _ = template.(string)
+	}
+	if template, ok := parsedMap["template"]; ok {
+		m.Template, _ = template.(string)
+	}
+	if variables, ok := parsedMap["variables"]; ok {
+		m.Variables, _ = variables.(map[string]interface{})
+	}
+}
+
+// MetadataParser is a an interface that must be satisfied by each parser
+type MetadataParser interface {
+	// Opening identifier
+	Opening() []byte
+
+	// Closing identifier
+	Closing() []byte
+
+	// Parse the metadata.
+	// Returns the remaining page contents (Markdown)
+	// after extracting metadata
+	Parse([]byte) ([]byte, error)
+
+	// Parsed metadata.
+	// Should be called after a call to Parse returns no error
+	Metadata() Metadata
+}
+
+// JSONMetadataParser is the MetdataParser for JSON
+type JSONMetadataParser struct {
+	metadata Metadata
+}
+
+// Parse the metadata
+func (j *JSONMetadataParser) Parse(b []byte) ([]byte, error) {
+	m := make(map[string]interface{})
+
+	// Read the preceding JSON object
+	decoder := json.NewDecoder(bytes.NewReader(b))
+	if err := decoder.Decode(&m); err != nil {
+		return b, err
+	}
+
+	j.metadata.load(m)
+
+	// Retrieve remaining bytes after decoding
+	buf := make([]byte, len(b))
+	n, err := decoder.Buffered().Read(buf)
+	if err != nil {
+		return b, err
+	}
+
+	return buf[:n], nil
+}
+
+// Parsed metadata.
+// Should be called after a call to Parse returns no error
+func (j *JSONMetadataParser) Metadata() Metadata {
+	return j.metadata
+}
+
+// Opening returns the opening identifier JSON metadata
+func (j *JSONMetadataParser) Opening() []byte {
+	return []byte("{")
+}
+
+// Closing returns the closing identifier JSON metadata
+func (j *JSONMetadataParser) Closing() []byte {
+	return []byte("}")
+}
+
+// TOMLMetadataParser is the MetadataParser for TOML
+type TOMLMetadataParser struct {
+	metadata Metadata
+}
+
+// Parse the metadata
+func (t *TOMLMetadataParser) Parse(b []byte) ([]byte, error) {
+	b, markdown, err := extractMetadata(t, b)
+	if err != nil {
+		return markdown, err
+	}
+	m := make(map[string]interface{})
+	if err := toml.Unmarshal(b, &m); err != nil {
+		return markdown, err
+	}
+	t.metadata.load(m)
+	return markdown, nil
+}
+
+// Parsed metadata.
+// Should be called after a call to Parse returns no error
+func (t *TOMLMetadataParser) Metadata() Metadata {
+	return t.metadata
+}
+
+// Opening returns the opening identifier TOML metadata
+func (t *TOMLMetadataParser) Opening() []byte {
+	return []byte("+++")
+}
+
+// Closing returns the closing identifier TOML metadata
+func (t *TOMLMetadataParser) Closing() []byte {
+	return []byte("+++")
+}
+
+// YAMLMetadataParser is the MetadataParser for YAML
+type YAMLMetadataParser struct {
+	metadata Metadata
+}
+
+// Parse the metadata
+func (y *YAMLMetadataParser) Parse(b []byte) ([]byte, error) {
+	b, markdown, err := extractMetadata(y, b)
+	if err != nil {
+		return markdown, err
+	}
+	m := make(map[string]interface{})
+	if err := yaml.Unmarshal(b, &m); err != nil {
+		return markdown, err
+	}
+	y.metadata.load(m)
+	return markdown, nil
+}
+
+// Parsed metadata.
+// Should be called after a call to Parse returns no error
+func (y *YAMLMetadataParser) Metadata() Metadata {
+	return y.metadata
+}
+
+// Opening returns the opening identifier YAML metadata
+func (y *YAMLMetadataParser) Opening() []byte {
+	return []byte("---")
+}
+
+// Closing returns the closing identifier YAML metadata
+func (y *YAMLMetadataParser) Closing() []byte {
+	return []byte("---")
+}
+
+// extractMetadata extracts metadata content from a page.
+// it returns the metadata, the remaining bytes (markdown),
+// and an error if any.
+// Useful for MetadataParser with defined identifiers (YAML, TOML)
+func extractMetadata(parser MetadataParser, b []byte) (metadata []byte, markdown []byte, err error) {
+	b = bytes.TrimSpace(b)
+	reader := bytes.NewBuffer(b)
+	scanner := bufio.NewScanner(reader)
+
+	// Read first line
+	if !scanner.Scan() {
+		// if no line is read,
+		// assume metadata not present
+		return nil, b, nil
+	}
+
+	line := bytes.TrimSpace(scanner.Bytes())
+	if !bytes.Equal(line, parser.Opening()) {
+		return nil, b, fmt.Errorf("Wrong identifier")
+	}
+
+	// buffer for metadata contents
+	buf := bytes.Buffer{}
+
+	// Read remaining lines until closing identifier is found
+	for scanner.Scan() {
+		line := scanner.Bytes()
+
+		// if closing identifier found
+		if bytes.Equal(bytes.TrimSpace(line), parser.Closing()) {
+
+			// get the scanner to return remaining bytes
+			scanner.Split(func(data []byte, atEOF bool) (int, []byte, error) {
+				return len(data), data, nil
+			})
+			// scan the remaining bytes
+			scanner.Scan()
+
+			return buf.Bytes(), scanner.Bytes(), nil
+		}
+		buf.Write(line)
+		buf.WriteString("\r\n")
+	}
+
+	// closing identifier not found
+	return buf.Bytes(), nil, fmt.Errorf("Metadata not closed. '%v' not found", string(parser.Closing()))
+}
+
+// findParser finds the parser using line that contains opening identifier
+func findParser(b []byte) MetadataParser {
+	var line []byte
+	// Read first line
+	if _, err := fmt.Fscanln(bytes.NewReader(b), &line); err != nil {
+		return nil
+	}
+	line = bytes.TrimSpace(line)
+	for _, parser := range parsers {
+		if bytes.Equal(parser.Opening(), line) {
+			return parser
+		}
+	}
+	return nil
+}

middleware/markdown/metadata_test.go

@@ -0,0 +1,165 @@
+package markdown
+
+import (
+	"bytes"
+	"fmt"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+var TOML = [4]string{`
+title = "A title"
+template = "default"
+[variables]
+name = "value"
+`,
+	`+++
+title = "A title"
+template = "default"
+[variables]
+name = "value"
++++
+Page content
+`,
+	`+++
+title = "A title"
+template = "default"
+[variables]
+name = "value"
+	`,
+	`title = "A title" template = "default" [variables] name = "value"`,
+}
+
+var YAML = [4]string{`
+title : A title
+template : default
+variables :
+- name : value
+`,
+	`---
+title : A title
+template : default
+variables :
+- name : value
+---
+Page content
+`,
+	`---
+title : A title
+template : default
+variables :
+- name : value
+`,
+	`title : A title template : default variables : name : value`,
+}
+var JSON = [4]string{`
+	"title" : "A title",
+	"template" : "default",
+	"variables" : {
+		"name" : "value"
+	}
+`,
+	`{
+	"title" : "A title",
+	"template" : "default",
+	"variables" : {
+		"name" : "value"
+	}
+}
+Page content
+`,
+	`
+{
+	"title" : "A title",
+	"template" : "default",
+	"variables" : {
+		"name" : "value"
+	}
+`,
+	`
+{{
+	"title" : "A title",
+	"template" : "default",
+	"variables" : {
+		"name" : "value"
+	}
+}
+	`,
+}
+
+func check(t *testing.T, err error) {
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestParsers(t *testing.T) {
+	expected := Metadata{
+		Title:     "A title",
+		Template:  "default",
+		Variables: map[string]interface{}{"name": "value"},
+	}
+	compare := func(m Metadata) bool {
+		if m.Title != expected.Title {
+			return false
+		}
+		if m.Template != expected.Template {
+			return false
+		}
+		for k, v := range m.Variables {
+			if v != expected.Variables[k] {
+				return false
+			}
+		}
+		return true
+	}
+
+	data := []struct {
+		parser   MetadataParser
+		testData [4]string
+		name     string
+	}{
+		{&JSONMetadataParser{}, JSON, "json"},
+		{&YAMLMetadataParser{}, YAML, "yaml"},
+		{&TOMLMetadataParser{}, TOML, "toml"},
+	}
+
+	for _, v := range data {
+		// metadata without identifiers
+		if _, err := v.parser.Parse([]byte(v.testData[0])); err == nil {
+			t.Fatalf("Expected error for invalid metadata for %v", v.name)
+		}
+
+		// metadata with identifiers
+		md, err := v.parser.Parse([]byte(v.testData[1]))
+		check(t, err)
+		if !compare(v.parser.Metadata()) {
+			t.Fatalf("Expected %v, found %v for %v", expected, v.parser.Metadata().Variables, v.name)
+		}
+		if "Page content" != strings.TrimSpace(string(md)) {
+			t.Fatalf("Expected %v, found %v for %v", "Page content", string(md), v.name)
+		}
+
+		var line []byte
+		fmt.Fscanln(bytes.NewReader([]byte(v.testData[1])), &line)
+		if parser := findParser(line); parser == nil {
+			t.Fatalf("Parser must be found for %v", v.name)
+		} else {
+			if reflect.TypeOf(parser) != reflect.TypeOf(v.parser) {
+				t.Fatalf("parsers not equal. %v != %v", reflect.TypeOf(parser), reflect.TypeOf(v.parser))
+			}
+		}
+
+		// metadata without closing identifier
+		if _, err := v.parser.Parse([]byte(v.testData[2])); err == nil {
+			t.Fatalf("Expected error for missing closing identifier for %v", v.name)
+		}
+
+		// invalid metadata
+		if md, err = v.parser.Parse([]byte(v.testData[3])); err == nil {
+			t.Fatalf("Expected error for invalid metadata for %v", v.name)
+		}
+	}
+
+}

middleware/markdown/process.go

@@ -0,0 +1,182 @@
+package markdown
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"text/template"
+
+	"github.com/russross/blackfriday"
+)
+
+const (
+	DefaultTemplate = "defaultTemplate"
+	StaticDir       = ".caddy_static"
+)
+
+// process processes the contents of a page.
+// It parses the metadata (if any) and uses the template (if found)
+func (md Markdown) process(c Config, requestPath string, b []byte) ([]byte, error) {
+	var metadata = Metadata{}
+	var markdown []byte
+	var err error
+
+	// find parser compatible with page contents
+	parser := findParser(b)
+
+	// if found, assume metadata present and parse.
+	if parser != nil {
+		markdown, err = parser.Parse(b)
+		if err != nil {
+			return nil, err
+		}
+		metadata = parser.Metadata()
+	}
+
+	// if template is not specified, check if Default template is set
+	if metadata.Template == "" {
+		if _, ok := c.Templates[DefaultTemplate]; ok {
+			metadata.Template = DefaultTemplate
+		}
+	}
+
+	// if template is set, load it
+	var tmpl []byte
+	if metadata.Template != "" {
+		if t, ok := c.Templates[metadata.Template]; ok {
+			tmpl, err = ioutil.ReadFile(t)
+		}
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// process markdown
+	markdown = blackfriday.Markdown(markdown, c.Renderer, 0)
+
+	// set it as body for template
+	metadata.Variables["markdown"] = string(markdown)
+
+	return md.processTemplate(c, requestPath, tmpl, metadata)
+}
+
+// processTemplate processes a template given a requestPath,
+// template (tmpl) and metadata
+func (md Markdown) processTemplate(c Config, requestPath string, tmpl []byte, metadata Metadata) ([]byte, error) {
+	// if template is not specified,
+	// use the default template
+	if tmpl == nil {
+		tmpl = defaultTemplate(c, metadata, requestPath)
+	}
+
+	// process the template
+	b := &bytes.Buffer{}
+	t, err := template.New("").Parse(string(tmpl))
+	if err != nil {
+		return nil, err
+	}
+	if err = t.Execute(b, metadata.Variables); err != nil {
+		return nil, err
+	}
+
+	// generate static page
+	if err = md.generatePage(c, requestPath, b.Bytes()); err != nil {
+		// if static page generation fails,
+		// nothing fatal, only log the error.
+		log.Println(err)
+	}
+
+	return b.Bytes(), nil
+
+}
+
+// generatePage generates a static html page from the markdown in content.
+func (md Markdown) generatePage(c Config, requestPath string, content []byte) error {
+	// should not happen,
+	// must be set on Markdown init.
+	if c.StaticDir == "" {
+		return fmt.Errorf("Static directory not set")
+	}
+
+	// if static directory is not existing, create it
+	if _, err := os.Stat(c.StaticDir); err != nil {
+		err := os.MkdirAll(c.StaticDir, os.FileMode(0755))
+		if err != nil {
+			return err
+		}
+	}
+
+	filePath := filepath.Join(c.StaticDir, requestPath)
+
+	// If it is index file, use the directory instead
+	if md.IsIndexFile(filepath.Base(requestPath)) {
+		filePath, _ = filepath.Split(filePath)
+	}
+
+	// Create the directory in case it is not existing
+	if err := os.MkdirAll(filePath, os.FileMode(0755)); err != nil {
+		return err
+	}
+
+	// generate index.html file in the directory
+	filePath = filepath.Join(filePath, "index.html")
+	err := ioutil.WriteFile(filePath, content, os.FileMode(0755))
+	if err != nil {
+		return err
+	}
+
+	c.StaticFiles[requestPath] = filePath
+	return nil
+}
+
+// defaultTemplate constructs a default template.
+func defaultTemplate(c Config, metadata Metadata, requestPath string) []byte {
+	var scripts, styles bytes.Buffer
+	for _, style := range c.Styles {
+		styles.WriteString(strings.Replace(cssTemplate, "{{url}}", style, 1))
+		styles.WriteString("\r\n")
+	}
+	for _, script := range c.Scripts {
+		scripts.WriteString(strings.Replace(jsTemplate, "{{url}}", script, 1))
+		scripts.WriteString("\r\n")
+	}
+
+	// Title is first line (length-limited), otherwise filename
+	title := metadata.Title
+	if title == "" {
+		title = filepath.Base(requestPath)
+		if body, _ := metadata.Variables["markdown"].([]byte); len(body) > 128 {
+			title = string(body[:128])
+		} else if len(body) > 0 {
+			title = string(body)
+		}
+	}
+
+	html := []byte(htmlTemplate)
+	html = bytes.Replace(html, []byte("{{title}}"), []byte(title), 1)
+	html = bytes.Replace(html, []byte("{{css}}"), styles.Bytes(), 1)
+	html = bytes.Replace(html, []byte("{{js}}"), scripts.Bytes(), 1)
+
+	return html
+}
+
+const (
+	htmlTemplate = `<!DOCTYPE html>
+<html>
+	<head>
+		<title>{{title}}</title>
+		<meta charset="utf-8">
+		{{css}}
+		{{js}}
+	</head>
+	<body>
+		{{.markdown}}
+	</body>
+</html>`
+	cssTemplate = `<link rel="stylesheet" href="{{url}}">`
+	jsTemplate  = `<script src="{{url}}"></script>`
+)