caddy/listen_unix.go

// Copyright 2015 Matthew Holt and The Caddy Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Even though the filename ends in _unix.go, we still have to specify the
// build constraint here, because the filename convention only works for
// literal GOOS values, and "unix" is a shortcut unique to build tags.
//go:build unix && !solaris

package caddy

import (
	"context"
	"errors"
	"io"
	"io/fs"
	"net"
	"os"
	"sync/atomic"
	"syscall"

	"go.uber.org/zap"
	"golang.org/x/sys/unix"
)

// reuseUnixSocket copies and reuses the unix domain socket (UDS) if we already
// have it open; if not, unlink it so we can have it. No-op if not a unix network.
func reuseUnixSocket(network, addr string) (any, error) {
	if !IsUnixNetwork(network) {
		return nil, nil
	}

	socketKey := listenerKey(network, addr)

	socket, exists := unixSockets[socketKey]
	if exists {
		// make copy of file descriptor
		socketFile, err := socket.File() // does dup() deep down
		if err != nil {
			return nil, err
		}

		// use copied fd to make new Listener or PacketConn, then replace
		// it in the map so that future copies always come from the most
		// recent fd (as the previous ones will be closed, and we'd get
		// "use of closed network connection" errors) -- note that we
		// preserve the *pointer* to the counter (not just the value) so
		// that all socket wrappers will refer to the same value
		switch unixSocket := socket.(type) {
		case *unixListener:
			ln, err := net.FileListener(socketFile)
			if err != nil {
				return nil, err
			}
			atomic.AddInt32(unixSocket.count, 1)
			unixSockets[socketKey] = &unixListener{ln.(*net.UnixListener), socketKey, unixSocket.count}

		case *unixConn:
			pc, err := net.FilePacketConn(socketFile)
			if err != nil {
				return nil, err
			}
			atomic.AddInt32(unixSocket.count, 1)
			unixSockets[socketKey] = &unixConn{pc.(*net.UnixConn), addr, socketKey, unixSocket.count}
		}

		return unixSockets[socketKey], nil
	}

	// from what I can tell after some quick research, it's quite common for programs to
	// leave their socket file behind after they close, so the typical pattern is to
	// unlink it before you bind to it -- this is often crucial if the last program using
	// it was killed forcefully without a chance to clean up the socket, but there is a
	// race, as the comment in net.UnixListener.close() explains... oh well, I guess?
	if err := syscall.Unlink(addr); err != nil && !errors.Is(err, fs.ErrNotExist) {
		return nil, err
	}

	return nil, nil
}

func listenReusable(ctx context.Context, lnKey string, network, address string, config net.ListenConfig) (any, error) {
	// wrap any Control function set by the user so we can also add our reusePort control without clobbering theirs
	oldControl := config.Control
	config.Control = func(network, address string, c syscall.RawConn) error {
		if oldControl != nil {
			if err := oldControl(network, address, c); err != nil {
				return err
			}
		}
		return reusePort(network, address, c)
	}

	// even though SO_REUSEPORT lets us bind the socket multiple times,
	// we still put it in the listenerPool so we can count how many
	// configs are using this socket; necessary to ensure we can know
	// whether to enforce shutdown delays, for example (see #5393).
	var ln io.Closer
	var err error
	switch network {
	case "udp", "udp4", "udp6", "unixgram":
		ln, err = config.ListenPacket(ctx, network, address)
	default:
		ln, err = config.Listen(ctx, network, address)
	}
	if err == nil {
		listenerPool.LoadOrStore(lnKey, nil)
	}

	// if new listener is a unix socket, make sure we can reuse it later
	// (we do our own "unlink on close" -- not required, but more tidy)
	one := int32(1)
	if unix, ok := ln.(*net.UnixListener); ok {
		unix.SetUnlinkOnClose(false)
		ln = &unixListener{unix, lnKey, &one}
		unixSockets[lnKey] = ln.(*unixListener)
	}

	// TODO: Not 100% sure this is necessary, but we do this for net.UnixListener in listen_unix.go, so...
	if unix, ok := ln.(*net.UnixConn); ok {
		ln = &unixConn{unix, address, lnKey, &one}
		unixSockets[lnKey] = ln.(*unixConn)
	}

	// lightly wrap the listener so that when it is closed,
	// we can decrement the usage pool counter
	switch specificLn := ln.(type) {
	case net.Listener:
		return deleteListener{specificLn, lnKey}, err
	case net.PacketConn:
		return deletePacketConn{specificLn, lnKey}, err
	}

	// other types, I guess we just return them directly
	return ln, err
}

// reusePort sets SO_REUSEPORT. Ineffective for unix sockets.
func reusePort(network, address string, conn syscall.RawConn) error {
	if IsUnixNetwork(network) {
		return nil
	}
	return conn.Control(func(descriptor uintptr) {
		if err := unix.SetsockoptInt(int(descriptor), unix.SOL_SOCKET, unixSOREUSEPORT, 1); err != nil {
			Log().Error("setting SO_REUSEPORT",
				zap.String("network", network),
				zap.String("address", address),
				zap.Uintptr("descriptor", descriptor),
				zap.Error(err))
		}
	})
}

type unixListener struct {
	*net.UnixListener
	mapKey string
	count  *int32 // accessed atomically
}

func (uln *unixListener) Close() error {
	newCount := atomic.AddInt32(uln.count, -1)
	if newCount == 0 {
		defer func() {
			addr := uln.Addr().String()
			unixSocketsMu.Lock()
			delete(unixSockets, uln.mapKey)
			unixSocketsMu.Unlock()
			_ = syscall.Unlink(addr)
		}()
	}
	return uln.UnixListener.Close()
}

type unixConn struct {
	*net.UnixConn
	filename string
	mapKey   string
	count    *int32 // accessed atomically
}

func (uc *unixConn) Close() error {
	newCount := atomic.AddInt32(uc.count, -1)
	if newCount == 0 {
		defer func() {
			unixSocketsMu.Lock()
			delete(unixSockets, uc.mapKey)
			unixSocketsMu.Unlock()
			_ = syscall.Unlink(uc.filename)
		}()
	}
	return uc.UnixConn.Close()
}

func (uc *unixConn) Unwrap() net.PacketConn {
	return uc.UnixConn
}

// unixSockets keeps track of the currently-active unix sockets
// so we can transfer their FDs gracefully during reloads.
var unixSockets = make(map[string]interface {
	File() (*os.File, error)
})

// deleteListener is a type that simply deletes itself
// from the listenerPool when it closes. It is used
// solely for the purpose of reference counting (i.e.
// counting how many configs are using a given socket).
type deleteListener struct {
	net.Listener
	lnKey string
}

func (dl deleteListener) Close() error {
	_, _ = listenerPool.Delete(dl.lnKey)
	return dl.Listener.Close()
}

// deletePacketConn is like deleteListener, but
// for net.PacketConns.
type deletePacketConn struct {
	net.PacketConn
	lnKey string
}

func (dl deletePacketConn) Close() error {
	_, _ = listenerPool.Delete(dl.lnKey)
	return dl.PacketConn.Close()
}

func (dl deletePacketConn) Unwrap() net.PacketConn {
	return dl.PacketConn
}