tls: Support distributed solving of the HTTP-01 challenge

Caddy can now obtain certificates when behind load balancers and/or in
fleet/cluster configurations, without needing any extra configuration.
The only requirement is sharing the same $CADDYPATH/acme folder.
This works with the HTTP challenge, whereas before the DNS challenge
was required. This commit allows one Caddy instance to initiate the
HTTP challenge and another to complete it.

When sharing that folder, certificate management is synchronized and
coordinated, without the Caddy instances needing to know about each
other. No load balancer reconfiguration should be required, either.

Currently, this is only supported when using FileStorage for TLS
storage (which is ~99.999% of users).

caddytls/client.go

@@ -137,21 +137,36 @@ var newACMEClient = func(config *Config, allowPrompts bool) (*ACMEClient, error)
 		// if config.AltTLSSNIPort != "" {
 		// 	useTLSSNIPort = config.AltTLSSNIPort
 		// }
-
-		// Always respect user's bind preferences by using config.ListenHost.
-		// NOTE(Sep'16): At time of writing, SetHTTPAddress() and SetTLSAddress()
-		// must be called before SetChallengeProvider(), since they reset the
-		// challenge provider back to the default one!
-		err := c.acmeClient.SetHTTPAddress(net.JoinHostPort(config.ListenHost, useHTTPPort))
-		if err != nil {
-			return nil, err
-		}
-		// TODO: tls-sni challenge was removed in January 2018, but a variant of it might return
-		// err = c.acmeClient.SetTLSAddress(net.JoinHostPort(config.ListenHost, useTLSSNIPort))
+		// err := c.acmeClient.SetTLSAddress(net.JoinHostPort(config.ListenHost, useTLSSNIPort))
 		// if err != nil {
 		// 	return nil, err
 		// }
 
+		// if using file storage, we can distribute the HTTP challenge across
+		// all instances sharing the acme folder; either way, we must still set
+		// the address for the default HTTP provider server
+		var useDistributedHTTPSolver bool
+		if storage, err := c.config.StorageFor(c.config.CAUrl); err == nil {
+			if _, ok := storage.(*FileStorage); ok {
+				useDistributedHTTPSolver = true
+			}
+		}
+		if useDistributedHTTPSolver {
+			c.acmeClient.SetChallengeProvider(acme.HTTP01, distributedHTTPSolver{
+				// being careful to respect user's listener bind preferences
+				httpProviderServer: acme.NewHTTPProviderServer(config.ListenHost, useHTTPPort),
+			})
+		} else {
+			// Always respect user's bind preferences by using config.ListenHost.
+			// NOTE(Sep'16): At time of writing, SetHTTPAddress() and SetTLSAddress()
+			// must be called before SetChallengeProvider() (see above), since they reset
+			// the challenge provider back to the default one! (still true in March 2018)
+			err := c.acmeClient.SetHTTPAddress(net.JoinHostPort(config.ListenHost, useHTTPPort))
+			if err != nil {
+				return nil, err
+			}
+		}
+
 		// TODO: tls-sni challenge was removed in January 2018, but a variant of it might return
 		// See if TLS challenge needs to be handled by our own facilities
 		// if caddy.HasListenerWithAddress(net.JoinHostPort(config.ListenHost, useTLSSNIPort)) {

caddytls/httphandler.go

@@ -16,12 +16,16 @@ package caddytls
 
 import (
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
 	"log"
 	"net/http"
 	"net/http/httputil"
 	"net/url"
+	"os"
 	"strings"
+
+	"github.com/xenolf/lego/acme"
 )
 
 const challengeBasePath = "/.well-known/acme-challenge"
@@ -38,6 +42,13 @@ func HTTPChallengeHandler(w http.ResponseWriter, r *http.Request, listenHost str
 	if DisableHTTPChallenge {
 		return false
 	}
+
+	// see if another instance started the HTTP challenge for this name
+	if tryDistributedChallengeSolver(w, r) {
+		return true
+	}
+
+	// otherwise, if we aren't getting the name, then ignore this challenge
 	if !namesObtaining.Has(r.Host) {
 		return false
 	}
@@ -70,3 +81,40 @@ func HTTPChallengeHandler(w http.ResponseWriter, r *http.Request, listenHost str
 
 	return true
 }
+
+// tryDistributedChallengeSolver checks to see if this challenge
+// request was initiated by another instance that shares file
+// storage, and attempts to complete the challenge for it. It
+// returns true if the challenge was handled; false otherwise.
+func tryDistributedChallengeSolver(w http.ResponseWriter, r *http.Request) bool {
+	filePath := distributedHTTPSolver{}.challengeTokensPath(r.Host)
+	f, err := os.Open(filePath)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			log.Printf("[ERROR][%s] Opening distributed challenge token file: %v", r.Host, err)
+		}
+		return false
+	}
+	defer f.Close()
+
+	var chalInfo challengeInfo
+	err = json.NewDecoder(f).Decode(&chalInfo)
+	if err != nil {
+		log.Printf("[ERROR][%s] Decoding challenge token file %s (corrupted?): %v", r.Host, filePath, err)
+		return false
+	}
+
+	// this part borrowed from xenolf/lego's built-in HTTP-01 challenge solver (March 2018)
+	challengeReqPath := acme.HTTP01ChallengePath(chalInfo.Token)
+	if r.URL.Path == challengeReqPath &&
+		strings.HasPrefix(r.Host, chalInfo.Domain) &&
+		r.Method == "GET" {
+		w.Header().Add("Content-Type", "text/plain")
+		w.Write([]byte(chalInfo.KeyAuth))
+		r.Close = true
+		log.Printf("[INFO][%s] Served key authentication", chalInfo.Domain)
+		return true
+	}
+
+	return false
+}

caddytls/tls.go

@@ -30,7 +30,12 @@ package caddytls
 
 import (
 	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
 	"net"
+	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/mholt/caddy"
@@ -128,6 +133,91 @@ func Revoke(host string) error {
 // 	return nil
 // }
 
+// distributedHTTPSolver allows the HTTP-01 challenge to be solved by
+// an instance other than the one which initiated it. This is useful
+// behind load balancers or in other cluster/fleet configurations.
+// The only requirement is that this (the initiating) instance share
+// the $CADDYPATH/acme folder with the instance that will complete
+// the challenge. Mounting the folder locally should be sufficient.
+//
+// Obviously, the instance which completes the challenge must be
+// serving on the HTTPChallengePort to receive and handle the request.
+// The HTTP server which receives it must check if a file exists, e.g.:
+// $CADDYPATH/acme/challenge_tokens/example.com.json, and if so,
+// decode it and use it to serve up the correct response. Caddy's HTTP
+// server does this by default.
+//
+// So as long as the folder is shared, this will just work. There are
+// no other requirements. The instances may be on other machines or
+// even other networks, as long as they share the folder as part of
+// the local file system.
+//
+// This solver works by persisting the token and keyauth information
+// to disk in the shared folder when the authorization is presented,
+// and then deletes it when it is cleaned up.
+type distributedHTTPSolver struct {
+	// The distributed HTTPS solver only works if an instance (either
+	// this one or another one) is already listening and serving on the
+	// HTTPChallengePort. If not -- for example: if this is the only
+	// instance, and it is just starting up and hasn't started serving
+	// yet -- then we still need a listener open with an HTTP server
+	// to handle the challenge request. Set this field to have the
+	// standard HTTPProviderServer open its listener for the duration
+	// of the challenge. Make sure to configure its listen address
+	// correctly.
+	httpProviderServer *acme.HTTPProviderServer
+}
+
+type challengeInfo struct {
+	Domain, Token, KeyAuth string
+}
+
+// Present adds the challenge certificate to the cache.
+func (dhs distributedHTTPSolver) Present(domain, token, keyAuth string) error {
+	if dhs.httpProviderServer != nil {
+		err := dhs.httpProviderServer.Present(domain, token, keyAuth)
+		if err != nil {
+			return fmt.Errorf("presenting with standard HTTP provider server: %v", err)
+		}
+	}
+
+	err := os.MkdirAll(dhs.challengeTokensBasePath(), 0755)
+	if err != nil {
+		return err
+	}
+
+	infoBytes, err := json.Marshal(challengeInfo{
+		Domain:  domain,
+		Token:   token,
+		KeyAuth: keyAuth,
+	})
+	if err != nil {
+		return err
+	}
+
+	return ioutil.WriteFile(dhs.challengeTokensPath(domain), infoBytes, 0644)
+}
+
+// CleanUp removes the challenge certificate from the cache.
+func (dhs distributedHTTPSolver) CleanUp(domain, token, keyAuth string) error {
+	if dhs.httpProviderServer != nil {
+		err := dhs.httpProviderServer.CleanUp(domain, token, keyAuth)
+		if err != nil {
+			log.Printf("[ERROR] Cleaning up standard HTTP provider server: %v", err)
+		}
+	}
+	return os.Remove(dhs.challengeTokensPath(domain))
+}
+
+func (dhs distributedHTTPSolver) challengeTokensPath(domain string) string {
+	domainFile := strings.Replace(strings.ToLower(domain), "*", "wildcard_", -1)
+	return filepath.Join(dhs.challengeTokensBasePath(), domainFile+".json")
+}
+
+func (dhs distributedHTTPSolver) challengeTokensBasePath() string {
+	return filepath.Join(caddy.AssetsPath(), "acme", "challenge_tokens")
+}
+
 // ConfigHolder is any type that has a Config; it presumably is
 // connected to a hostname and port on which it is serving.
 type ConfigHolder interface {