From 5447ec5bdddb31df51df13f70e5abf8187ac9893 Mon Sep 17 00:00:00 2001
From: Ramkumar Chinchani <rchincha@cisco.com>
Date: Mon, 18 Nov 2019 11:28:52 -0800
Subject: [PATCH] ldap: improve recovery when connection failures

---
 errors/errors.go |  1 +
 pkg/api/ldap.go  | 50 +++++++++++++++++++++++++++++++++++-------------
 2 files changed, 38 insertions(+), 13 deletions(-)

diff --git a/errors/errors.go b/errors/errors.go
index 3d9aa32b..08be4653 100644
--- a/errors/errors.go
+++ b/errors/errors.go
@@ -19,5 +19,6 @@ var (
 	ErrBadUser             = errors.New("ldap: non-existent user")
 	ErrEntriesExceeded     = errors.New("ldap: too many entries returned")
 	ErrLDAPEmptyPassphrase = errors.New("ldap: empty passphrase")
+	ErrLDAPBadConn         = errors.New("ldap: bad connection")
 	ErrLDAPConfig          = errors.New("config: invalid LDAP configuration")
 )
diff --git a/pkg/api/ldap.go b/pkg/api/ldap.go
index e6eee3b0..332d0729 100644
--- a/pkg/api/ldap.go
+++ b/pkg/api/ldap.go
@@ -4,6 +4,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
+	"time"
 
 	"github.com/anuvu/zot/errors"
 	"github.com/jtblin/go-ldap-client"
@@ -11,6 +12,8 @@ import (
 	goldap "gopkg.in/ldap.v2"
 )
 
+const maxRetries = 8
+
 type LDAPClient struct {
 	ldap.LDAPClient
 	subtreeSearch bool
@@ -69,6 +72,17 @@ func (lc *LDAPClient) Connect() error {
 	return nil
 }
 
+func sleepAndRetry(retries, maxRetries int) bool {
+	if retries > maxRetries {
+		return false
+	}
+	if retries < maxRetries {
+		time.Sleep(time.Duration(retries) * time.Second) // gradually backoff
+		return true
+	}
+	return false
+}
+
 // Authenticate authenticates the user against the ldap backend.
 func (lc *LDAPClient) Authenticate(username, password string) (bool, map[string]string, error) {
 	if password == "" {
@@ -76,21 +90,31 @@ func (lc *LDAPClient) Authenticate(username, password string) (bool, map[string]
 		return false, nil, errors.ErrLDAPEmptyPassphrase
 	}
 
-	err := lc.Connect()
-	if err != nil {
-		return false, nil, err
+	connected := false
+	for retries := 0; !connected && sleepAndRetry(retries, maxRetries); retries++ {
+		err := lc.Connect()
+		if err != nil {
+			continue
+		}
+
+		// First bind with a read only user
+		if lc.BindDN != "" && lc.BindPassword != "" {
+			err := lc.Conn.Bind(lc.BindDN, lc.BindPassword)
+			if err != nil {
+				lc.log.Error().Err(err).Str("bindDN", lc.BindDN).Msg("bind failed")
+				// clean up the cached conn, so we can retry
+				lc.Conn.Close()
+				lc.Conn = nil
+				continue
+			}
+		}
+		connected = true
 	}
 
-	// First bind with a read only user
-	if lc.BindDN != "" && lc.BindPassword != "" {
-		err := lc.Conn.Bind(lc.BindDN, lc.BindPassword)
-		if err != nil {
-			lc.log.Error().Err(err).Str("bindDN", lc.BindDN).Msg("bind failed")
-			// clean up the cached conn, so we can retry
-			lc.Conn.Close()
-			lc.Conn = nil
-			return false, nil, err
-		}
+	// exhausted all retries?
+	if !connected {
+		lc.log.Error().Err(errors.ErrLDAPBadConn).Msg("exhausted all retries")
+		return false, nil, errors.ErrLDAPBadConn
 	}
 
 	attributes := append(lc.Attributes, "dn")