letsencrypt: Basic renewal failover and better error handling

config/letsencrypt/renew.go

@@ -17,8 +17,10 @@ import (
 func keepCertificatesRenewed(configs []server.Config) {
 	ticker := time.Tick(renewInterval)
 	for range ticker {
-		if err := processCertificateRenewal(configs); err != nil {
+		if errs := processCertificateRenewal(configs); len(errs) > 0 {
-			log.Printf("[ERROR] cert renewal: %v", err)
+			for _, err := range errs {
+				log.Printf("[ERROR] cert renewal: %v\n", err)
+			}
 		}
 	}
 }
@@ -26,11 +28,12 @@ func keepCertificatesRenewed(configs []server.Config) {
 // checkCertificateRenewal loops through all configured
 // sites and looks for certificates to renew. Nothing is mutated
 // through this function. The changes happen directly on disk.
-func processCertificateRenewal(configs []server.Config) error {
+func processCertificateRenewal(configs []server.Config) []error {
+	var errs []error
 	log.Print("[INFO] Processing certificate renewals...")
 
 	for _, cfg := range configs {
-		// Check if this entry is TLS enabled and managed by LE
+		// Host must be TLS-enabled and have assets managed by LE
 		if !cfg.TLS.Enabled || !existingCertAndKey(cfg.Host) {
 			continue
 		}
@@ -38,11 +41,13 @@ func processCertificateRenewal(configs []server.Config) error {
 		// Read the certificate and get the NotAfter time.
 		certBytes, err := ioutil.ReadFile(storage.SiteCertFile(cfg.Host))
 		if err != nil {
-			return err
+			errs = append(errs, err)
+			continue // still have to check other certificates
 		}
 		expTime, err := acme.GetPEMCertExpiration(certBytes)
 		if err != nil {
-			return err
+			errs = append(errs, err)
+			continue
 		}
 
 		// The time returned from the certificate is always in UTC.
@@ -50,23 +55,26 @@ func processCertificateRenewal(configs []server.Config) error {
 		// Directly convert it to days for the following checks.
 		daysLeft := int(expTime.Sub(time.Now().UTC()).Hours() / 24)
 
-		// Renew on two or less days remaining.
+		// Renew with a week or less remaining.
-		if daysLeft <= 2 {
+		if daysLeft <= 7 {
-			log.Printf("[WARN] There are %d days left on the certificate of %s. Trying to renew now.", daysLeft, cfg.Host)
+			log.Printf("[INFO] There are %d days left on the certificate of %s. Trying to renew now.", daysLeft, cfg.Host)
 			client, err := newClient(getEmail(cfg))
 			if err != nil {
-				return err
+				errs = append(errs, err)
+				continue
 			}
 
 			// Read metadata
 			metaBytes, err := ioutil.ReadFile(storage.SiteMetaFile(cfg.Host))
 			if err != nil {
-				return err
+				errs = append(errs, err)
+				continue
 			}
 
 			privBytes, err := ioutil.ReadFile(storage.SiteKeyFile(cfg.Host))
 			if err != nil {
-				return err
+				errs = append(errs, err)
+				continue
 			}
 
 			var certMeta acme.CertificateResource
@@ -78,17 +86,20 @@ func processCertificateRenewal(configs []server.Config) error {
 			// TODO: revokeOld should be an option in the caddyfile
 			newCertMeta, err := client.RenewCertificate(certMeta, true)
 			if err != nil {
-				return err
+				time.Sleep(10 * time.Second)
+				newCertMeta, err = client.RenewCertificate(certMeta, true)
+				if err != nil {
+					errs = append(errs, err)
+					continue
+				}
 			}
 
 			saveCertsAndKeys([]acme.CertificateResource{newCertMeta})
-		}
+		} else if daysLeft <= 14 {
-
 			// Warn on 14 days remaining
-		if daysLeft <= 14 {
+			log.Printf("[WARN] There are %d days left on the certificate for %s. Will renew when 7 days remain.\n", daysLeft, cfg.Host)
-			log.Printf("[WARN] There are %d days left on the certificate of %s. Will renew on two days left.\n", daysLeft, cfg.Host)
 		}
 	}
 
-	return nil
+	return errs
 }