core: Refactor and improve listener logic (#5089)

* core: Refactor, improve listener logic

Deprecate:
- caddy.Listen
- caddy.ListenTimeout
- caddy.ListenPacket

Prefer caddy.NetworkAddress.Listen() instead.

Change:
- caddy.ListenQUIC (hopefully to remove later)
- caddy.ListenerFunc signature (add context and ListenConfig)

- Don't emit Alt-Svc header advertising h3 over HTTP/3

- Use quic.ListenEarly instead of quic.ListenEarlyAddr; this gives us
more flexibility (e.g. possibility of HTTP/3 over UDS) but also
introduces a new issue:
https://github.com/lucas-clemente/quic-go/issues/3560#issuecomment-1258959608

- Unlink unix socket before and after use

* Appease the linter

* Keep ListenAll

modules/caddyhttp/app.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"net"
 	"net/http"
 	"strconv"
 	"sync"
@@ -387,10 +388,11 @@ func (app *App) Start() error {
 			for portOffset := uint(0); portOffset < listenAddr.PortRangeSize(); portOffset++ {
 				// create the listener for this socket
 				hostport := listenAddr.JoinHostPort(portOffset)
-				ln, err := caddy.ListenTimeout(listenAddr.Network, hostport, time.Duration(srv.KeepAliveInterval))
+				lnAny, err := listenAddr.Listen(app.ctx, portOffset, net.ListenConfig{KeepAlive: time.Duration(srv.KeepAliveInterval)})
 				if err != nil {
-					return fmt.Errorf("%s: listening on %s: %v", listenAddr.Network, hostport, err)
+					return fmt.Errorf("listening on %s: %v", listenAddr.At(portOffset), err)
 				}
+				ln := lnAny.(net.Listener)
 
 				// wrap listener before TLS (up to the TLS placeholder wrapper)
 				var lnWrapperIdx int
@@ -409,10 +411,27 @@ func (app *App) Start() error {
 					ln = tls.NewListener(ln, tlsCfg)
 
 					// enable HTTP/3 if configured
-					if srv.protocol("h3") && !listenAddr.IsUnixNetwork() {
-						app.logger.Info("enabling HTTP/3 listener", zap.String("addr", hostport))
-						if err := srv.serveHTTP3(hostport, tlsCfg); err != nil {
-							return err
+					if srv.protocol("h3") {
+						// Can't serve HTTP/3 on the same socket as HTTP/1 and 2 because it uses
+						// a different transport mechanism... which is fine, but the OS doesn't
+						// differentiate between a SOCK_STREAM file and a SOCK_DGRAM file; they
+						// are still one file on the system. So even though "unixpacket" and
+						// "unixgram" are different network types just as "tcp" and "udp" are,
+						// the OS will not let us use the same file as both STREAM and DGRAM.
+						if len(srv.Protocols) > 1 && listenAddr.IsUnixNetwork() {
+							app.logger.Warn("HTTP/3 disabled because Unix can't multiplex STREAM and DGRAM on same socket",
+								zap.String("file", hostport))
+							for i := range srv.Protocols {
+								if srv.Protocols[i] == "h3" {
+									srv.Protocols = append(srv.Protocols[:i], srv.Protocols[i+1:]...)
+									break
+								}
+							}
+						} else {
+							app.logger.Info("enabling HTTP/3 listener", zap.String("addr", hostport))
+							if err := srv.serveHTTP3(listenAddr.At(portOffset), tlsCfg); err != nil {
+								return err
+							}
 						}
 					}
 				}
@@ -424,11 +443,10 @@ func (app *App) Start() error {
 
 				// if binding to port 0, the OS chooses a port for us;
 				// but the user won't know the port unless we print it
-				if listenAddr.StartPort == 0 && listenAddr.EndPort == 0 {
+				if !listenAddr.IsUnixNetwork() && listenAddr.StartPort == 0 && listenAddr.EndPort == 0 {
 					app.logger.Info("port 0 listener",
 						zap.String("input_address", lnAddr),
-						zap.String("actual_address", ln.Addr().String()),
-					)
+						zap.String("actual_address", ln.Addr().String()))
 				}
 
 				app.logger.Debug("starting server loop",
@@ -533,6 +551,18 @@ func (app *App) Stop() error {
 		if server.h3server == nil {
 			return
 		}
+
+		// TODO: we have to manually close our listeners because quic-go won't
+		// close listeners it didn't create along with the server itself...
+		// see https://github.com/lucas-clemente/quic-go/issues/3560
+		for _, el := range server.h3listeners {
+			if err := el.Close(); err != nil {
+				app.logger.Error("HTTP/3 listener close",
+					zap.Error(err),
+					zap.String("address", el.LocalAddr().String()))
+			}
+		}
+
 		// TODO: CloseGracefully, once implemented upstream (see https://github.com/lucas-clemente/quic-go/issues/2103)
 		if err := server.h3server.Close(); err != nil {
 			app.logger.Error("HTTP/3 server shutdown",

modules/caddyhttp/matchers.go

@@ -156,7 +156,9 @@ type (
 	MatchHeaderRE map[string]*MatchRegexp
 
 	// MatchProtocol matches requests by protocol. Recognized values are
-	// "http", "https", and "grpc".
+	// "http", "https", and "grpc" for broad protocol matches, or specific
+	// HTTP versions can be specified like so: "http/1", "http/1.1",
+	// "http/2", "http/3", or minimum versions: "http/2+", etc.
 	MatchProtocol string
 
 	// MatchRemoteIP matches requests by client IP (or CIDR range).

modules/caddyhttp/server.go

@@ -170,9 +170,10 @@ type Server struct {
 	errorLogger  *zap.Logger
 	ctx          caddy.Context
 
-	server    *http.Server
-	h3server  *http3.Server
-	addresses []caddy.NetworkAddress
+	server      *http.Server
+	h3server    *http3.Server
+	h3listeners []net.PacketConn // TODO: we have to hold these because quic-go won't close listeners it didn't create
+	addresses   []caddy.NetworkAddress
 
 	shutdownAt   time.Time
 	shutdownAtMu *sync.RWMutex
@@ -193,9 +194,11 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		atomic.AddInt64(&s.activeRequests, 1)
 		defer atomic.AddInt64(&s.activeRequests, -1)
 
-		err := s.h3server.SetQuicHeaders(w.Header())
-		if err != nil {
-			s.logger.Error("setting HTTP/3 Alt-Svc header", zap.Error(err))
+		if r.ProtoMajor < 3 {
+			err := s.h3server.SetQuicHeaders(w.Header())
+			if err != nil {
+				s.logger.Error("setting HTTP/3 Alt-Svc header", zap.Error(err))
+			}
 		}
 	}
 
@@ -493,8 +496,27 @@ func (s *Server) findLastRouteWithHostMatcher() int {
 // serveHTTP3 creates a QUIC listener, configures an HTTP/3 server if
 // not already done, and then uses that server to serve HTTP/3 over
 // the listener, with Server s as the handler.
-func (s *Server) serveHTTP3(hostport string, tlsCfg *tls.Config) error {
-	h3ln, err := caddy.ListenQUIC(hostport, tlsCfg, &s.activeRequests)
+func (s *Server) serveHTTP3(addr caddy.NetworkAddress, tlsCfg *tls.Config) error {
+	switch addr.Network {
+	case "unix":
+		addr.Network = "unixgram"
+	case "tcp":
+		addr.Network = "udp"
+	case "tcp4":
+		addr.Network = "udp4"
+	case "tcp6":
+		addr.Network = "udp6"
+	default:
+		return fmt.Errorf("unsure what network to use for HTTP/3 given network type: %s", addr.Network)
+	}
+
+	lnAny, err := addr.Listen(s.ctx, 0, net.ListenConfig{})
+	if err != nil {
+		return err
+	}
+	ln := lnAny.(net.PacketConn)
+
+	h3ln, err := caddy.ListenQUIC(ln, tlsCfg, &s.activeRequests)
 	if err != nil {
 		return fmt.Errorf("starting HTTP/3 QUIC listener: %v", err)
 	}
@@ -512,6 +534,8 @@ func (s *Server) serveHTTP3(hostport string, tlsCfg *tls.Config) error {
 		}
 	}
 
+	s.h3listeners = append(s.h3listeners, lnAny.(net.PacketConn))
+
 	//nolint:errcheck
 	go s.h3server.ServeListener(h3ln)