From 479c6114201ef29aacd97b4922ad6b700cfb7d4e Mon Sep 17 00:00:00 2001 From: Abiola Ibrahim Date: Fri, 1 May 2015 03:11:01 +0100 Subject: [PATCH] Implementation of Git middleware Defaults path to site root. --- config/middleware.go | 2 + middleware/git/doc.go | 67 +++++++++ middleware/git/git.go | 156 ++++++++++++++++++++ middleware/git/gitclient.go | 277 ++++++++++++++++++++++++++++++++++++ 4 files changed, 502 insertions(+) create mode 100644 middleware/git/doc.go create mode 100644 middleware/git/git.go create mode 100644 middleware/git/gitclient.go diff --git a/config/middleware.go b/config/middleware.go index d144cc6a..41af1826 100644 --- a/config/middleware.go +++ b/config/middleware.go @@ -7,6 +7,7 @@ import ( "github.com/mholt/caddy/middleware/errors" "github.com/mholt/caddy/middleware/extensions" "github.com/mholt/caddy/middleware/fastcgi" + "github.com/mholt/caddy/middleware/git" "github.com/mholt/caddy/middleware/gzip" "github.com/mholt/caddy/middleware/headers" "github.com/mholt/caddy/middleware/log" @@ -48,6 +49,7 @@ func init() { register("ext", extensions.New) register("basicauth", basicauth.New) register("proxy", proxy.New) + register("git", git.New) register("fastcgi", fastcgi.New) register("websocket", websockets.New) register("markdown", markdown.New) diff --git a/middleware/git/doc.go b/middleware/git/doc.go new file mode 100644 index 00000000..ae416c4d --- /dev/null +++ b/middleware/git/doc.go @@ -0,0 +1,67 @@ +// Package git is the middleware that pull sites from git repo +// +// Caddyfile Syntax : +// git repo path { +// repo +// path +// branch +// key +// interval +// } +// repo - git repository +// compulsory. Both ssh (e.g. git@github.com:user/project.git) +// and https(e.g. https://github.com/user/project) are supported. +// Can be specified in either config block or top level +// +// path - directory to pull into +// optional. Defaults to site root. +// +// branch - git branch or tag +// optional. Defaults to master +// +// key - path to private ssh key +// optional. Required for private repositories. e.g. /home/user/.ssh/id_rsa +// +// interval- interval between git pulls in seconds +// optional. Defaults to 3600 (1 Hour). +// +// Examples : +// +// public repo pulled into site root +// git github.com/user/myproject +// +// public repo pulled into mysite +// git https://github.com/user/myproject mysite +// +// private repo pulled into mysite with tag v1.0 and interval of 1 day +// git { +// repo git@github.com:user/myproject +// branch v1.0 +// path mysite +// key /home/user/.ssh/id_rsa +// interval 86400 # 1 day +// } +// +// Caddyfile with private git repo and php support via fastcgi. +// path defaults to /var/www/html/myphpsite as specified in root config. +// +// 0.0.0.0:8080 +// +// git { +// repo git@github.com:user/myphpsite +// key /home/user/.ssh/id_rsa +// interval 86400 # 1 day +// } +// +// fastcgi / 127.0.0.1:9000 php +// +// root /var/www/html/myphpsite +// +// A pull is first attempted after initialization. Afterwards, a pull is attempted +// after request to server and if time taken since last successful pull is higher than interval. +// +// After the first successful pull (should be during initialization except an error occurs), +// subsequent pulls are done in background and do not impact request time. +// +// Note: private repositories are currently only supported and tested on Linux and OSX +package git diff --git a/middleware/git/git.go b/middleware/git/git.go new file mode 100644 index 00000000..2823d6e0 --- /dev/null +++ b/middleware/git/git.go @@ -0,0 +1,156 @@ +package git + +import ( + "fmt" + "github.com/mholt/caddy/middleware" + "net/http" + "net/url" + "runtime" + "strconv" + "strings" + "time" +) + +// Git represents a middleware instance that pulls git repository. +type Git struct { + Next middleware.Handler + Repo *Repo +} + +// ServeHTTP satisfies the middleware.Handler interface. +func (g Git) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) { + if err := g.Repo.Pull(); err != nil { + return 500, err + } + return g.Next.ServeHTTP(w, r) +} + +// New creates a new instance of git middleware. +func New(c middleware.Controller) (middleware.Middleware, error) { + repo, err := parse(c) + if err != nil { + return nil, err + } + err = repo.Pull() + return func(next middleware.Handler) middleware.Handler { + return Git{Next: next, Repo: repo} + }, err +} + +func parse(c middleware.Controller) (*Repo, error) { + repo := &Repo{Branch: "master", Interval: DefaultInterval, Path: c.Root()} + + for c.Next() { + args := c.RemainingArgs() + + switch len(args) { + case 2: + repo.Path = args[1] + fallthrough + case 1: + repo.Url = args[0] + } + + for c.NextBlock() { + switch c.Val() { + case "repo": + if !c.NextArg() { + return nil, c.ArgErr() + } + repo.Url = c.Val() + case "path": + if !c.NextArg() { + return nil, c.ArgErr() + } + repo.Path = c.Val() + case "branch": + if !c.NextArg() { + return nil, c.ArgErr() + } + repo.Branch = c.Val() + case "key": + if !c.NextArg() { + return nil, c.ArgErr() + } + repo.KeyPath = c.Val() + case "interval": + if !c.NextArg() { + return nil, c.ArgErr() + } + t, _ := strconv.Atoi(c.Val()) + if t > 0 { + repo.Interval = time.Duration(t) * time.Second + } + } + } + } + + // if repo is not specified, return error + if repo.Url == "" { + return nil, c.ArgErr() + } + + // if private key is not specified, convert repository url to https + // to avoid ssh authentication + // else validate git url + // Note: private key support not yet available on Windows + var err error + if repo.KeyPath == "" { + repo.Url, repo.Host, err = sanitizeHttp(repo.Url) + } else { + repo.Url, repo.Host, err = sanitizeGit(repo.Url) + // TODO add Windows support for private repos + if runtime.GOOS == "windows" { + return nil, fmt.Errorf("Private repository not yet supported on Windows") + } + } + + if err != nil { + return nil, err + } + + // validate git availability in PATH + if err = initGit(); err != nil { + return nil, err + } + + return repo, prepare(repo) +} + +// sanitizeHttp cleans up repository url and converts to https format +// if currently in ssh format. +// Returns sanitized url, hostName (e.g. github.com, bitbucket.com) +// and possible error +func sanitizeHttp(repoUrl string) (string, string, error) { + url, err := url.Parse(repoUrl) + if err != nil { + return "", "", err + } + + if url.Host == "" && strings.HasPrefix(url.Path, "git@") { + url.Path = url.Path[len("git@"):] + i := strings.Index(url.Path, ":") + if i < 0 { + return "", "", fmt.Errorf("Invalid git url %s", repoUrl) + } + url.Host = url.Path[:i] + url.Path = "/" + url.Path[i+1:] + } + + repoUrl = "https://" + url.Host + url.Path + return repoUrl, url.Host, nil +} + +// sanitizeGit cleans up repository url and validate ssh format. +// Returns sanitized url, hostName (e.g. github.com, bitbucket.com) +// and possible error +func sanitizeGit(repoUrl string) (string, string, error) { + repoUrl = strings.TrimSpace(repoUrl) + if !strings.HasPrefix(repoUrl, "git@") || strings.Index(repoUrl, ":") < len("git@a:") { + return "", "", fmt.Errorf("Invalid git url %s", repoUrl) + } + hostUrl := repoUrl[len("git@"):] + i := strings.Index(hostUrl, ":") + host := hostUrl[:i] + return repoUrl, host, nil +} diff --git a/middleware/git/gitclient.go b/middleware/git/gitclient.go new file mode 100644 index 00000000..b62069f3 --- /dev/null +++ b/middleware/git/gitclient.go @@ -0,0 +1,277 @@ +package git + +import ( + "fmt" + "io/ioutil" + "log" + "os" + "os/exec" + "strings" + "sync" + "time" +) + +// DefaultInterval is the minimum interval to delay before +// requesting another git pull +const DefaultInterval time.Duration = time.Hour * 1 + +// gitBinary holds the absolute path to git executable +var gitBinary string + +// initMutex prevents parallel attempt to validate +// git availability in PATH +var initMutex sync.Mutex = sync.Mutex{} + +// Repo is the structure that holds required information +// of a git repository. +type Repo struct { + Url string // Repository URL + Path string // Directory to pull to + Host string // Git domain host e.g. github.com + Branch string // Git branch + KeyPath string // Path to private ssh key + Interval time.Duration // Interval between pulls + pulled bool // true if there is a successful pull + lastPull time.Time // time of the last successful pull + sync.Mutex +} + +// Pull requests a repository pull. +// If it has been performed previously, it returns +// and requests another pull in background. +// Otherwise it waits until the pull is done. +func (r *Repo) Pull() error { + // if site is not pulled, pull + if !r.pulled { + return pull(r) + } + + // request pull in background + go pull(r) + return nil +} + +// pull performs git clone, or git pull if repository exists +func pull(r *Repo) error { + r.Lock() + defer r.Unlock() + // if it is less than interval since last pull, return + if time.Since(r.lastPull) <= r.Interval { + return nil + } + + params := []string{"clone", "-b", r.Branch, r.Url, r.Path} + if r.pulled { + params = []string{"pull", "origin", r.Branch} + } + + // if key is specified, pull using ssh key + if r.KeyPath != "" { + return pullWithKey(r, params) + } + + cmd := exec.Command(gitBinary, params...) + cmd.Env = os.Environ() + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + if r.pulled { + cmd.Dir = r.Path + } + + var err error + if err = cmd.Start(); err != nil { + return err + } + + if err = cmd.Wait(); err == nil { + r.pulled = true + r.lastPull = time.Now() + log.Printf("%v pulled.\n", r.Url) + } + + return err +} + +// pullWithKey performs git clone or git pull if repository exists. +// It is used for private repositories and requires an ssh key. +// Note: currently only limited to Linux and OSX. +func pullWithKey(r *Repo, params []string) error { + var gitSsh, script *os.File + // ensure temporary files deleted after usage + defer func() { + if gitSsh != nil { + os.Remove(gitSsh.Name()) + } + if script != nil { + os.Remove(script.Name()) + } + }() + + var err error + // write git.sh script to temp file + gitSsh, err = writeScriptFile(gitWrapperScript(gitBinary)) + if err != nil { + return err + } + + // write git clone bash script to file + script, err = writeScriptFile(bashScript(gitSsh.Name(), r, params)) + if err != nil { + return err + } + + // execute the git clone bash script + cmd := exec.Command(script.Name()) + cmd.Env = os.Environ() + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + if r.pulled { + cmd.Dir = r.Path + } + + if err = cmd.Start(); err != nil { + return err + } + + if err = cmd.Wait(); err == nil { + r.pulled = true + r.lastPull = time.Now() + log.Printf("%v pulled.\n", r.Url) + } + return err +} + +// prepare prepares for a git pull +// and validates the configured directory +func prepare(r *Repo) error { + // check if directory exists or is empty + // if not, create directory + fs, err := ioutil.ReadDir(r.Path) + if err != nil || len(fs) == 0 { + return os.MkdirAll(r.Path, os.FileMode(0755)) + } + + // validate git repo + isGit := false + for _, f := range fs { + if f.IsDir() && f.Name() == ".git" { + isGit = true + break + } + } + + if isGit { + // check if same repository + var repoUrl string + if repoUrl, err = getRepoUrl(r.Path); err == nil && repoUrl == r.Url { + r.pulled = true + return nil + } + if err != nil { + return fmt.Errorf("Cannot retrieve repo url for %v Error: %v", r.Path, err) + } + return fmt.Errorf("Another git repo '%v' exists at %v", repoUrl, r.Path) + } + return fmt.Errorf("Cannot git clone into %v, directory not empty.", r.Path) +} + +// getRepoUrl retrieves remote origin url for the git repository at path +func getRepoUrl(path string) (string, error) { + args := []string{"config", "--get", "remote.origin.url"} + + _, err := os.Stat(path) + if err != nil { + return "", err + } + + cmd := exec.Command(gitBinary, args...) + cmd.Dir = path + output, err := cmd.Output() + if err != nil { + return "", err + } + + return strings.TrimSpace(string(output)), nil +} + +// initGit validates git installation and locates the git executable +// binary in PATH +func initGit() error { + // prevent concurrent call + initMutex.Lock() + defer initMutex.Unlock() + + // if validation has been done before and binary located in + // PATH, return. + if gitBinary != "" { + return nil + } + + // locate git binary in path + var err error + gitBinary, err = exec.LookPath("git") + return err + +} + +// writeScriptFile writes content to a temporary file. +// It changes the temporary file mode to executable and +// closes it to prepare it for execution. +func writeScriptFile(content []byte) (file *os.File, err error) { + if file, err = ioutil.TempFile("", "caddy"); err != nil { + return nil, err + } + if _, err = file.Write(content); err != nil { + return nil, err + } + if err = file.Chmod(os.FileMode(0755)); err != nil { + return nil, err + } + return file, file.Close() +} + +// gitWrapperScript forms content for git.sh script +var gitWrapperScript = func(gitBinary string) []byte { + return []byte(fmt.Sprintf(`#!/bin/bash + +# The MIT License (MIT) +# Copyright (c) 2013 Alvin Abad + +if [ $# -eq 0 ]; then + echo "Git wrapper script that can specify an ssh-key file +Usage: + git.sh -i ssh-key-file git-command + " + exit 1 +fi + +# remove temporary file on exit +trap 'rm -f /tmp/.git_ssh.$$' 0 + +if [ "$1" = "-i" ]; then + SSH_KEY=$2; shift; shift + echo "ssh -i $SSH_KEY \$@" > /tmp/.git_ssh.$$ + chmod +x /tmp/.git_ssh.$$ + export GIT_SSH=/tmp/.git_ssh.$$ +fi + +# in case the git command is repeated +[ "$1" = "git" ] && shift + +# Run the git command +%v "$@" + +`, gitBinary)) +} + +// bashScript forms content of bash script to clone or update a repo using ssh +var bashScript = func(gitShPath string, repo *Repo, params []string) []byte { + return []byte(fmt.Sprintf(`#!/bin/bash + +mkdir -p ~/.ssh; +touch ~/.ssh/known_hosts; +ssh-keyscan -t rsa,dsa %v 2>&1 | sort -u - ~/.ssh/known_hosts > ~/.ssh/tmp_hosts; +cat ~/.ssh/tmp_hosts >> ~/.ssh/known_hosts; +%v -i %v %v; +`, repo.Host, gitShPath, repo.KeyPath, strings.Join(params, " "))) +}