From 63af971a1ec5c6d5aac8660184c389e4a02633f0 Mon Sep 17 00:00:00 2001 From: Sai Krishna Kumar Reddy YADAMAKANTI Date: Tue, 29 Sep 2026 15:07:24 +0000 Subject: [PATCH] QUIC: support BPF reload and binary upgrade The SO_COOKIE dependency is removed. QUIC BPF dispatching now uses separate sockets for new and established connections: listener sockets handle new connections, while per-worker sockets handle established ones. Server connection IDs encode a random worker key, which BPF uses to route established packets directly to the correct worker socket. New connections are routed through listen keys using a dual slot scheme that enables both reload and binary upgrade. On reload, new workers register new sockets and keys while old workers continue receiving established connections through their encoded keys. On binary upgrade, the new master uses the opposite slot, and exiting masters clear their entries so traffic shifts to the surviving instance. Co-authored-by: Roman Arutyunyan --- auto/modules | 4 +- auto/options | 2 - auto/os/linux | 20 +- src/core/ngx_bpf.c | 2 +- src/core/ngx_connection.c | 4 +- src/event/ngx_event.h | 1 + src/event/quic/bpf/makefile | 5 +- .../quic/bpf/ngx_quic_reuseport_helper.c | 112 ++- src/event/quic/ngx_event_quic.h | 6 + src/event/quic/ngx_event_quic_bpf.c | 932 ++++++++++++++---- src/event/quic/ngx_event_quic_bpf.h | 20 + src/event/quic/ngx_event_quic_bpf_code.c | 203 +++- src/event/quic/ngx_event_quic_connection.h | 3 + src/event/quic/ngx_event_quic_connid.c | 21 +- src/event/quic/ngx_event_quic_output.c | 1 - src/event/quic/ngx_event_quic_udp.c | 23 +- src/os/unix/ngx_process_cycle.c | 3 +- 17 files changed, 1079 insertions(+), 283 deletions(-) create mode 100644 src/event/quic/ngx_event_quic_bpf.h diff --git a/auto/modules b/auto/modules index 0f037b278..99e7b4f3b 100644 --- a/auto/modules +++ b/auto/modules @@ -1455,11 +1455,11 @@ if [ $USE_OPENSSL_QUIC = YES ]; then . auto/module - if [ $QUIC_BPF = YES -a $SO_COOKIE_FOUND = YES ]; then + if [ $QUIC_BPF = YES ]; then ngx_module_type=CORE ngx_module_name=ngx_quic_bpf_module ngx_module_incs= - ngx_module_deps= + ngx_module_deps=src/event/quic/ngx_event_quic_bpf.h ngx_module_srcs="src/event/quic/ngx_event_quic_bpf.c \ src/event/quic/ngx_event_quic_bpf_code.c" ngx_module_libs= diff --git a/auto/options b/auto/options index 332d74efb..7220174a9 100644 --- a/auto/options +++ b/auto/options @@ -178,8 +178,6 @@ USE_GEOIP=NO NGX_GOOGLE_PERFTOOLS=NO NGX_CPP_TEST=NO -SO_COOKIE_FOUND=NO - NGX_LIBATOMIC=NO NGX_CPU_CACHE_LINE= diff --git a/auto/os/linux b/auto/os/linux index bc0556b3a..df8d03451 100644 --- a/auto/os/linux +++ b/auto/os/linux @@ -249,7 +249,10 @@ ngx_feature_test="union bpf_attr attr = { 0 }; attr.map_flags = 0; attr.map_type = BPF_MAP_TYPE_SOCKHASH; + syscall(__NR_bpf, 0, &attr, 0); + attr.map_flags = 0; + attr.map_type = BPF_MAP_TYPE_ARRAY; syscall(__NR_bpf, 0, &attr, 0);" . auto/feature @@ -263,23 +266,6 @@ if [ $ngx_found = yes ]; then fi -ngx_feature="SO_COOKIE" -ngx_feature_name="NGX_HAVE_SO_COOKIE" -ngx_feature_run=no -ngx_feature_incs="#include - $NGX_INCLUDE_INTTYPES_H" -ngx_feature_path= -ngx_feature_libs= -ngx_feature_test="socklen_t optlen = sizeof(uint64_t); - uint64_t cookie; - getsockopt(0, SOL_SOCKET, SO_COOKIE, &cookie, &optlen)" -. auto/feature - -if [ $ngx_found = yes ]; then - SO_COOKIE_FOUND=YES -fi - - # UDP segmentation offloading ngx_feature="UDP_SEGMENT" diff --git a/src/core/ngx_bpf.c b/src/core/ngx_bpf.c index 363a02c7d..d08b7c41c 100644 --- a/src/core/ngx_bpf.c +++ b/src/core/ngx_bpf.c @@ -7,7 +7,7 @@ #include #include -#define NGX_BPF_LOGBUF_SIZE (16 * 1024) +#define NGX_BPF_LOGBUF_SIZE (32 * 1024) static ngx_inline int diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c index 7ea7706e5..faf987bbf 100644 --- a/src/core/ngx_connection.c +++ b/src/core/ngx_connection.c @@ -1142,7 +1142,9 @@ ngx_close_listening_sockets(ngx_cycle_t *cycle) for (i = 0; i < cycle->listening.nelts; i++) { #if (NGX_QUIC) - if (ls[i].quic) { + if (ls[i].quic + && (!ls[i].reuseport || !ngx_quic_bpf_enabled(cycle))) + { continue; } #endif diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h index deac04e7b..5196353bc 100644 --- a/src/event/ngx_event.h +++ b/src/event/ngx_event.h @@ -73,6 +73,7 @@ struct ngx_event_s { /* to test on worker exit */ unsigned channel:1; unsigned resolver:1; + unsigned quic:1; unsigned cancelable:1; diff --git a/src/event/quic/bpf/makefile b/src/event/quic/bpf/makefile index 277aa3151..e8e449fc8 100644 --- a/src/event/quic/bpf/makefile +++ b/src/event/quic/bpf/makefile @@ -1,4 +1,7 @@ -CFLAGS=-O2 -Wall + +# Copyright (C) Nginx, Inc. + +CFLAGS=-O2 -Wall $(MAKE_CFLAGS) LICENSE=BSD diff --git a/src/event/quic/bpf/ngx_quic_reuseport_helper.c b/src/event/quic/bpf/ngx_quic_reuseport_helper.c index cf7a24c5a..51f9923a5 100644 --- a/src/event/quic/bpf/ngx_quic_reuseport_helper.c +++ b/src/event/quic/bpf/ngx_quic_reuseport_helper.c @@ -54,17 +54,23 @@ char _license[] SEC("license") = LICENSE; ((__u64)(p)[6] << 8) | \ ((__u64)(p)[7])) +#define ngx_quic_bpf_listen_key(worker) \ + (((__u64) 0xFF << 56) | ((__u64) (worker) & 0xFFFFFFFFFFFFULL)) + + /* * actual map object is created by the "bpf" system call, * all pointers to this variable are replaced by the bpf loader */ -extern int ngx_quic_sockmap; +struct {} ngx_quic_sockmap SEC(".maps"); +struct {} ngx_quic_worker_counts SEC(".maps"); SEC(PROGNAME) int ngx_quic_select_socket_by_dcid(struct sk_reuseport_md *ctx) { - int rc; + int rc, worker_idx; + __u32 wc_key, *wc0, *wc1, n, m; __u64 key; size_t offset; unsigned char *start, *end, *dcid, byte, buf[NGX_QUIC_SERVER_CID_LEN]; @@ -98,7 +104,7 @@ int ngx_quic_select_socket_by_dcid(struct sk_reuseport_md *ctx) } if (byte != NGX_QUIC_SERVER_CID_LEN) { - return SK_PASS; + goto new; } } @@ -118,30 +124,84 @@ int ngx_quic_select_socket_by_dcid(struct sk_reuseport_md *ctx) key = ngx_quic_parse_uint64(dcid); - rc = bpf_sk_select_reuseport(ctx, &ngx_quic_sockmap, &key, 0); - - switch (rc) { - case 0: - debugmsg("nginx quic socket selected by key 0x%llx", key); - return SK_PASS; - - /* kernel returns positive error numbers, errno.h defines positive */ - case -ENOENT: - debugmsg("nginx quic default route for key 0x%llx", key); - /* let the default reuseport logic decide which socket to choose */ - return SK_PASS; - - default: - debugmsg("nginx quic bpf_sk_select_reuseport err: %d key 0x%llx", - rc, key); - goto failed; + if ((key >> 56) == 0xFF) { + goto new; } + rc = bpf_sk_select_reuseport(ctx, &ngx_quic_sockmap, &key, 0); + + if (rc == 0) { + debugmsg("nginx quic worker socket selected by dcid"); + return SK_PASS; + } + + if (rc != -ENOENT) { + debugmsg("nginx quic bpf_sk_select_reuseport() failed: %d", rc); + return SK_DROP; + } + +new: + + wc_key = 0; + wc0 = bpf_map_lookup_elem(&ngx_quic_worker_counts, &wc_key); + if (wc0 == NULL) { + debugmsg("nginx quic worker count 0 undefined"); + return SK_DROP; + } + + wc_key = 1; + wc1 = bpf_map_lookup_elem(&ngx_quic_worker_counts, &wc_key); + if (wc1 == NULL) { + debugmsg("nginx quic worker count 1 undefined"); + return SK_DROP; + } + + n = *wc0 > *wc1 ? *wc0 : *wc1; + + if (n == 0) { + debugmsg("nginx quic no active workers"); + return SK_DROP; + } + + worker_idx = ctx->hash % n; + key = ngx_quic_bpf_listen_key(worker_idx); + + rc = bpf_sk_select_reuseport(ctx, &ngx_quic_sockmap, &key, 0); + + if (rc == 0) { + debugmsg("nginx quic listener socket selected worker index:%d", + worker_idx); + return SK_PASS; + } + + if (rc != -ENOENT) { + debugmsg("nginx quic bpf_sk_select_reuseport() failed: %d", rc); + return SK_DROP; + } + + m = *wc0 < *wc1 ? *wc0 : *wc1; + + if (m && m != n) { + worker_idx = ctx->hash % m; + key = ngx_quic_bpf_listen_key(worker_idx); + + rc = bpf_sk_select_reuseport(ctx, &ngx_quic_sockmap, &key, 0); + + if (rc == 0) { + debugmsg("nginx quic listener socket selected " + "worker index:%d (fallback)", worker_idx); + return SK_PASS; + } + + debugmsg("nginx quic bpf_sk_select_reuseport() fallback " + "failed: %d", rc); + } + + return SK_DROP; + failed: - /* - * SK_DROP will generate ICMP, but we may want to process "invalid" packet - * in userspace quic to investigate further and finally react properly - * (maybe ignore, maybe send something in response or close connection) - */ - return SK_PASS; + + debugmsg("nginx quic bad datagram"); + + return SK_DROP; } diff --git a/src/event/quic/ngx_event_quic.h b/src/event/quic/ngx_event_quic.h index 4f899ec0a..e26673ac6 100644 --- a/src/event/quic/ngx_event_quic.h +++ b/src/event/quic/ngx_event_quic.h @@ -142,4 +142,10 @@ ngx_int_t ngx_quic_get_packet_dcid(ngx_log_t *log, u_char *data, size_t len, ngx_int_t ngx_quic_derive_key(ngx_log_t *log, const char *label, ngx_str_t *secret, ngx_str_t *salt, u_char *out, size_t len); +#if (NGX_QUIC_BPF) +ngx_uint_t ngx_quic_bpf_enabled(ngx_cycle_t *cycle); +#else +#define ngx_quic_bpf_enabled(c) 0 +#endif + #endif /* _NGX_EVENT_QUIC_H_INCLUDED_ */ diff --git a/src/event/quic/ngx_event_quic_bpf.c b/src/event/quic/ngx_event_quic_bpf.c index ab024ad56..2488d9c75 100644 --- a/src/event/quic/ngx_event_quic_bpf.c +++ b/src/event/quic/ngx_event_quic_bpf.c @@ -6,6 +6,8 @@ #include #include +#include +#include #define NGX_QUIC_BPF_VARNAME "NGINX_BPF_MAPS" @@ -23,42 +25,67 @@ #define ngx_core_get_conf(cycle) \ (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module) +#define ngx_quic_bpf_listen_key(worker) \ + (((uint64_t) 0xFF << 56) \ + | ((uint64_t) (worker) & 0xFFFFFFFFFFFFULL)) + typedef struct { ngx_queue_t queue; - int map_fd; + + int sock_map; + int worker_counts_map; struct sockaddr *sockaddr; socklen_t socklen; - ngx_uint_t unused; /* unsigned unused:1; */ + + ngx_array_t listening; } ngx_quic_sock_group_t; +typedef struct { + ngx_socket_t fd; + uint64_t key; + ngx_listening_t *listening; + ngx_connection_t *connection; +} ngx_quic_bpf_listening_t; + + typedef struct { ngx_flag_t enabled; ngx_uint_t map_size; + ngx_uint_t master_index; + u_char *env; ngx_queue_t groups; /* of ngx_quic_sock_group_t */ } ngx_quic_bpf_conf_t; static void *ngx_quic_bpf_create_conf(ngx_cycle_t *cycle); +static char *ngx_quic_bpf_init_conf(ngx_cycle_t *cycle, void *conf); static ngx_int_t ngx_quic_bpf_module_init(ngx_cycle_t *cycle); +static void ngx_quic_bpf_exit_master(ngx_cycle_t *cycle); static void ngx_quic_bpf_cleanup(void *data); static ngx_inline void ngx_quic_bpf_close(ngx_log_t *log, int fd, const char *name); +static ngx_inline ngx_int_t ngx_quic_bpf_map_update(ngx_log_t *log, int fd, + const void *key, const void *value, const char *name); static ngx_quic_sock_group_t *ngx_quic_bpf_find_group(ngx_quic_bpf_conf_t *bcf, ngx_listening_t *ls); static ngx_quic_sock_group_t *ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle, - struct sockaddr *sa, socklen_t socklen); + ngx_listening_t *ls); static ngx_quic_sock_group_t *ngx_quic_bpf_create_group(ngx_cycle_t *cycle, ngx_listening_t *ls); +static ngx_int_t ngx_quic_bpf_inherit_fd(ngx_cycle_t *cycle, int fd); static ngx_quic_sock_group_t *ngx_quic_bpf_get_group(ngx_cycle_t *cycle, ngx_listening_t *ls); static ngx_int_t ngx_quic_bpf_group_add_socket(ngx_cycle_t *cycle, ngx_listening_t *ls); -static uint64_t ngx_quic_bpf_socket_key(ngx_fd_t fd, ngx_log_t *log); +static ngx_int_t ngx_quic_bpf_add_worker_socket(ngx_cycle_t *cycle, + ngx_quic_sock_group_t *grp, ngx_listening_t *ls); +static ngx_int_t ngx_quic_bpf_publish_workers(ngx_cycle_t *cycle, + ngx_quic_sock_group_t *grp, ngx_uint_t worker_count); static ngx_int_t ngx_quic_bpf_export_maps(ngx_cycle_t *cycle); static ngx_int_t ngx_quic_bpf_import_maps(ngx_cycle_t *cycle); @@ -82,7 +109,7 @@ static ngx_command_t ngx_quic_bpf_commands[] = { static ngx_core_module_t ngx_quic_bpf_module_ctx = { ngx_string("quic_bpf"), ngx_quic_bpf_create_conf, - NULL + ngx_quic_bpf_init_conf }; @@ -97,7 +124,7 @@ ngx_module_t ngx_quic_bpf_module = { NULL, /* init thread */ NULL, /* exit thread */ NULL, /* exit process */ - NULL, /* exit master */ + ngx_quic_bpf_exit_master, /* exit master */ NGX_MODULE_V1_PADDING }; @@ -105,6 +132,8 @@ ngx_module_t ngx_quic_bpf_module = { static void * ngx_quic_bpf_create_conf(ngx_cycle_t *cycle) { + size_t len; + u_char *env; ngx_quic_bpf_conf_t *bcf; bcf = ngx_pcalloc(cycle->pool, sizeof(ngx_quic_bpf_conf_t)); @@ -115,20 +144,66 @@ ngx_quic_bpf_create_conf(ngx_cycle_t *cycle) bcf->enabled = NGX_CONF_UNSET; bcf->map_size = NGX_CONF_UNSET_UINT; + /* + * preserve environment variable value before it may be reset by + * some module, i.e. perl + */ + env = (u_char *) getenv(NGX_QUIC_BPF_VARNAME); + if (env != NULL) { + len = ngx_strlen(env); + bcf->env = ngx_pnalloc(cycle->pool, len + 1); + if (bcf->env == NULL) { + return NULL; + } + + ngx_memcpy(bcf->env, env, len + 1); + } + ngx_queue_init(&bcf->groups); return bcf; } +static char * +ngx_quic_bpf_init_conf(ngx_cycle_t *cycle, void *conf) +{ + ngx_quic_bpf_conf_t *bcf = conf; + ngx_quic_bpf_conf_t *obcf; + + ngx_conf_init_value(bcf->enabled, 0); + + if (cycle->old_cycle->conf_ctx == NULL) { + return NGX_CONF_OK; + } + + obcf = ngx_quic_bpf_get_conf(cycle->old_cycle); + if (obcf == NULL) { + return NGX_CONF_OK; + } + + if (obcf->enabled != bcf->enabled) { + ngx_log_error(NGX_LOG_WARN, cycle->log, 0, + "cannot change \"quic_bpf\" after reload, ignoring"); + bcf->enabled = obcf->enabled; + } + + bcf->master_index = obcf->master_index; + + return NGX_CONF_OK; +} + + static ngx_int_t ngx_quic_bpf_module_init(ngx_cycle_t *cycle) { - ngx_uint_t i; - ngx_listening_t *ls; - ngx_core_conf_t *ccf; - ngx_pool_cleanup_t *cln; - ngx_quic_bpf_conf_t *bcf; + ngx_uint_t i; + ngx_queue_t *q; + ngx_listening_t *ls; + ngx_core_conf_t *ccf; + ngx_pool_cleanup_t *cln; + ngx_quic_bpf_conf_t *bcf; + ngx_quic_sock_group_t *grp; if (ngx_test_config) { /* @@ -141,9 +216,21 @@ ngx_quic_bpf_module_init(ngx_cycle_t *cycle) ccf = ngx_core_get_conf(cycle); bcf = ngx_quic_bpf_get_conf(cycle); - ngx_conf_init_value(bcf->enabled, 0); + if (!bcf->enabled) { - bcf->map_size = ccf->worker_processes * 4; + if (ngx_inherited && ngx_is_init_cycle(cycle->old_cycle) + && bcf->env != NULL) + { + ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, + "\"quic_bpf\" cannot be disabled " + "during binary upgrade"); + return NGX_ERROR; + } + + return NGX_OK; + } + + bcf->map_size = ccf->worker_processes * 32; cln = ngx_pool_cleanup_add(cycle->pool, 0); if (cln == NULL) { @@ -169,6 +256,29 @@ ngx_quic_bpf_module_init(ngx_cycle_t *cycle) } } + q = ngx_queue_head(&bcf->groups); + + while (q != ngx_queue_sentinel(&bcf->groups)) { + + grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue); + q = ngx_queue_next(q); + + if (grp->listening.nelts == 0) { + + ngx_quic_bpf_close(cycle->log, grp->sock_map, "sockmap"); + ngx_quic_bpf_close(cycle->log, grp->worker_counts_map, + "worker_counts"); + ngx_queue_remove(&grp->queue); + continue; + } + + if (ngx_quic_bpf_publish_workers(cycle, grp, ccf->worker_processes) + != NGX_OK) + { + goto failed; + } + } + if (ngx_quic_bpf_export_maps(cycle) != NGX_OK) { goto failed; } @@ -203,13 +313,33 @@ failed: } +ngx_uint_t +ngx_quic_bpf_enabled(ngx_cycle_t *cycle) +{ + ngx_quic_bpf_conf_t *bcf; + + bcf = ngx_quic_bpf_get_conf(cycle); + + if (bcf == NULL) { + return 0; + } + + return bcf->enabled; +} + + static void ngx_quic_bpf_cleanup(void *data) { ngx_quic_bpf_conf_t *bcf = (ngx_quic_bpf_conf_t *) data; - ngx_queue_t *q; - ngx_quic_sock_group_t *grp; + ngx_uint_t i; + ngx_log_t *log; + ngx_queue_t *q; + ngx_quic_sock_group_t *grp; + ngx_quic_bpf_listening_t *bls; + + log = ngx_cycle->log; for (q = ngx_queue_head(&bcf->groups); q != ngx_queue_sentinel(&bcf->groups); @@ -217,7 +347,51 @@ ngx_quic_bpf_cleanup(void *data) { grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue); - ngx_quic_bpf_close(ngx_cycle->log, grp->map_fd, "map"); + bls = grp->listening.elts; + + for (i = 0; i < grp->listening.nelts; i++) { + if (bls[i].fd != (ngx_socket_t) -1) { + if (ngx_close_socket(bls[i].fd) == -1) { + ngx_log_error(NGX_LOG_ALERT, log, ngx_socket_errno, + ngx_close_socket_n " failed"); + } + } + } + + ngx_quic_bpf_close(log, grp->sock_map, "sockmap"); + ngx_quic_bpf_close(log, grp->worker_counts_map, "worker_counts"); + } +} + + +static void +ngx_quic_bpf_exit_master(ngx_cycle_t *cycle) +{ + uint32_t key, value; + ngx_queue_t *q; + ngx_quic_bpf_conf_t *bcf; + ngx_quic_sock_group_t *grp; + + bcf = ngx_quic_bpf_get_conf(cycle); + if (!bcf->enabled) { + return; + } + + key = bcf->master_index; + value = 0; + + for (q = ngx_queue_head(&bcf->groups); + q != ngx_queue_sentinel(&bcf->groups); + q = ngx_queue_next(q)) + { + grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue); + + if (ngx_quic_bpf_map_update(cycle->log, grp->worker_counts_map, + &key, &value, "worker_counts") + != NGX_OK) + { + continue; + } } } @@ -225,6 +399,10 @@ ngx_quic_bpf_cleanup(void *data) static ngx_inline void ngx_quic_bpf_close(ngx_log_t *log, int fd, const char *name) { + if (fd == -1) { + return; + } + if (close(fd) != -1) { return; } @@ -234,6 +412,21 @@ ngx_quic_bpf_close(ngx_log_t *log, int fd, const char *name) } +static ngx_inline ngx_int_t +ngx_quic_bpf_map_update(ngx_log_t *log, int fd, const void *key, + const void *value, const char *name) +{ + if (ngx_bpf_map_update(fd, key, value, BPF_ANY) != -1) { + return NGX_OK; + } + + ngx_log_error(NGX_LOG_ERR, log, ngx_errno, + "quic bpf failed to update %s map fd:%d", name, fd); + + return NGX_ERROR; +} + + static ngx_quic_sock_group_t * ngx_quic_bpf_find_group(ngx_quic_bpf_conf_t *bcf, ngx_listening_t *ls) { @@ -259,8 +452,7 @@ ngx_quic_bpf_find_group(ngx_quic_bpf_conf_t *bcf, ngx_listening_t *ls) static ngx_quic_sock_group_t * -ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle, struct sockaddr *sa, - socklen_t socklen) +ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle, ngx_listening_t *ls) { ngx_quic_bpf_conf_t *bcf; ngx_quic_sock_group_t *grp; @@ -272,12 +464,18 @@ ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle, struct sockaddr *sa, return NULL; } - grp->socklen = socklen; - grp->sockaddr = ngx_palloc(cycle->pool, socklen); - if (grp->sockaddr == NULL) { + grp->sock_map = -1; + grp->worker_counts_map = -1; + + grp->sockaddr = ls->sockaddr; + grp->socklen = ls->socklen; + + if (ngx_array_init(&grp->listening, cycle->pool, 1, + sizeof(ngx_quic_bpf_listening_t)) + != NGX_OK) + { return NULL; } - ngx_memcpy(grp->sockaddr, sa, socklen); ngx_queue_insert_tail(&bcf->groups, &grp->queue); @@ -288,47 +486,51 @@ ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle, struct sockaddr *sa, static ngx_quic_sock_group_t * ngx_quic_bpf_create_group(ngx_cycle_t *cycle, ngx_listening_t *ls) { - int progfd, failed, flags, rc; + int progfd, failed; ngx_quic_bpf_conf_t *bcf; ngx_quic_sock_group_t *grp; bcf = ngx_quic_bpf_get_conf(cycle); - if (!bcf->enabled) { - return NULL; - } - - grp = ngx_quic_bpf_alloc_group(cycle, ls->sockaddr, ls->socklen); + grp = ngx_quic_bpf_alloc_group(cycle, ls); if (grp == NULL) { return NULL; } - grp->map_fd = ngx_bpf_map_create(cycle->log, BPF_MAP_TYPE_SOCKHASH, - sizeof(uint64_t), sizeof(uint64_t), - bcf->map_size, 0); - if (grp->map_fd == -1) { + grp->sock_map = ngx_bpf_map_create(cycle->log, + BPF_MAP_TYPE_SOCKHASH, + sizeof(uint64_t), + sizeof(uint64_t), + bcf->map_size, 0); + if (grp->sock_map == -1) { goto failed; } - flags = fcntl(grp->map_fd, F_GETFD); - if (flags == -1) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, errno, - "quic bpf getfd failed"); - goto failed; - } - - /* need to inherit map during binary upgrade after exec */ - flags &= ~FD_CLOEXEC; - - rc = fcntl(grp->map_fd, F_SETFD, flags); - if (rc == -1) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, errno, - "quic bpf setfd failed"); + if (ngx_quic_bpf_inherit_fd(cycle, grp->sock_map) != NGX_OK) { goto failed; } ngx_bpf_program_link(&ngx_quic_reuseport_helper, - "ngx_quic_sockmap", grp->map_fd); + "ngx_quic_sockmap", grp->sock_map); + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + "quic bpf sockmap created fd:%d", + grp->sock_map); + + grp->worker_counts_map = ngx_bpf_map_create(cycle->log, + BPF_MAP_TYPE_ARRAY, + sizeof(uint32_t), + sizeof(uint32_t), 2, 0); + if (grp->worker_counts_map == -1) { + goto failed; + } + + if (ngx_quic_bpf_inherit_fd(cycle, grp->worker_counts_map) != NGX_OK) { + goto failed; + } + + ngx_bpf_program_link(&ngx_quic_reuseport_helper, + "ngx_quic_worker_counts", grp->worker_counts_map); progfd = ngx_bpf_load_program(cycle->log, &ngx_quic_reuseport_helper); if (progfd < 0) { @@ -352,15 +554,12 @@ ngx_quic_bpf_create_group(ngx_cycle_t *cycle, ngx_listening_t *ls) goto failed; } - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, - "quic bpf sockmap created fd:%d", grp->map_fd); return grp; failed: - if (grp->map_fd != -1) { - ngx_quic_bpf_close(cycle->log, grp->map_fd, "map"); - } + ngx_quic_bpf_close(cycle->log, grp->sock_map, "sockmap"); + ngx_quic_bpf_close(cycle->log, grp->worker_counts_map, "worker_counts"); ngx_queue_remove(&grp->queue); @@ -368,6 +567,30 @@ failed: } +static ngx_int_t +ngx_quic_bpf_inherit_fd(ngx_cycle_t *cycle, int fd) +{ + int flags; + + flags = fcntl(fd, F_GETFD); + if (flags == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "quic bpf fcntl(F_GETFD) failed"); + return NGX_ERROR; + } + + flags &= ~FD_CLOEXEC; + + if (fcntl(fd, F_SETFD, flags) == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "quic bpf fcntl(F_SETFD) failed"); + return NGX_ERROR; + } + + return NGX_OK; +} + + static ngx_quic_sock_group_t * ngx_quic_bpf_get_group(ngx_cycle_t *cycle, ngx_listening_t *ls) { @@ -392,89 +615,346 @@ ngx_quic_bpf_get_group(ngx_cycle_t *cycle, ngx_listening_t *ls) return ngx_quic_bpf_create_group(cycle, ls); } - grp = ngx_quic_bpf_alloc_group(cycle, ls->sockaddr, ls->socklen); + grp = ngx_quic_bpf_alloc_group(cycle, ls); if (grp == NULL) { return NULL; } - grp->map_fd = dup(ogrp->map_fd); - if (grp->map_fd == -1) { + grp->sock_map = dup(ogrp->sock_map); + if (grp->sock_map == -1) { ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, - "quic bpf failed to duplicate bpf map descriptor"); - - ngx_queue_remove(&grp->queue); - - return NULL; + "quic bpf failed to duplicate sockmap"); + goto failed; } - ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0, - "quic bpf sockmap fd duplicated old:%d new:%d", - ogrp->map_fd, grp->map_fd); + grp->worker_counts_map = dup(ogrp->worker_counts_map); + if (grp->worker_counts_map == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "quic bpf failed to duplicate worker counts map"); + goto failed; + } return grp; + +failed: + + ngx_quic_bpf_close(cycle->log, grp->sock_map, "sockmap"); + ngx_quic_bpf_close(cycle->log, grp->worker_counts_map, "worker_counts"); + + ngx_queue_remove(&grp->queue); + + return NULL; } static ngx_int_t -ngx_quic_bpf_group_add_socket(ngx_cycle_t *cycle, ngx_listening_t *ls) +ngx_quic_bpf_group_add_socket(ngx_cycle_t *cycle, ngx_listening_t *ls) { - uint64_t cookie; - ngx_quic_bpf_conf_t *bcf; + uint64_t key, value; ngx_quic_sock_group_t *grp; - bcf = ngx_quic_bpf_get_conf(cycle); - grp = ngx_quic_bpf_get_group(cycle, ls); - if (grp == NULL) { - if (!bcf->enabled) { - return NGX_OK; - } - return NGX_ERROR; } - grp->unused = 0; - - cookie = ngx_quic_bpf_socket_key(ls->fd, cycle->log); - if (cookie == (uint64_t) NGX_ERROR) { + if (ngx_quic_bpf_add_worker_socket(cycle, grp, ls) != NGX_OK) { return NGX_ERROR; } - /* map[cookie] = socket; for use in kernel helper */ - if (ngx_bpf_map_update(grp->map_fd, &cookie, &ls->fd, BPF_ANY) == -1) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, - "quic bpf failed to update socket map key=%xL", cookie); + key = ngx_quic_bpf_listen_key(ls->worker); + value = ls->fd; + + if (ngx_quic_bpf_map_update(cycle->log, grp->sock_map, + &key, &value, "listen") + != NGX_OK) + { return NGX_ERROR; } - ngx_log_debug4(NGX_LOG_DEBUG_EVENT, cycle->log, 0, - "quic bpf sockmap fd:%d add socket:%d cookie:0x%xL worker:%ui", - grp->map_fd, ls->fd, cookie, ls->worker); - - /* do not inherit this socket */ - ls->ignore = 1; - return NGX_OK; } -static uint64_t -ngx_quic_bpf_socket_key(ngx_fd_t fd, ngx_log_t *log) +static ngx_int_t +ngx_quic_bpf_publish_workers(ngx_cycle_t *cycle, ngx_quic_sock_group_t *grp, + ngx_uint_t worker_count) { - uint64_t cookie; - socklen_t optlen; + uint32_t key; + uint32_t value; + ngx_quic_bpf_conf_t *bcf; - optlen = sizeof(cookie); + bcf = ngx_quic_bpf_get_conf(cycle); - if (getsockopt(fd, SOL_SOCKET, SO_COOKIE, &cookie, &optlen) == -1) { - ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, - "quic bpf getsockopt(SO_COOKIE) failed"); + key = bcf->master_index; + value = worker_count; - return (ngx_uint_t) NGX_ERROR; + if (ngx_quic_bpf_map_update(cycle->log, grp->worker_counts_map, + &key, &value, "worker_counts") + != NGX_OK) + { + return NGX_ERROR; } - return cookie; + return NGX_OK; +} + + +static ngx_int_t +ngx_quic_bpf_add_worker_socket(ngx_cycle_t *cycle, ngx_quic_sock_group_t *grp, + ngx_listening_t *ls) +{ + int value; + uint64_t key, map_value; + ngx_uint_t i, n, map_updated; + ngx_socket_t s; + ngx_quic_bpf_listening_t *bls; + + s = ngx_socket(ls->sockaddr->sa_family, SOCK_DGRAM, 0); + if (s == (ngx_socket_t) -1) { + ngx_log_error(NGX_LOG_ERR, cycle->log, ngx_socket_errno, + "quic bpf " ngx_socket_n " failed"); + return NGX_ERROR; + } + + map_updated = 0; + + if (ngx_nonblocking(s) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf " ngx_nonblocking_n " failed"); + goto failed; + } + + if (fcntl(s, F_SETFD, FD_CLOEXEC) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "quic bpf fcntl(FD_CLOEXEC) failed"); + goto failed; + } + + value = 1; + + if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(SO_REUSEADDR) failed"); + goto failed; + } + + if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(SO_REUSEPORT) failed"); + goto failed; + } + + if (ls->rcvbuf != -1) { + if (setsockopt(s, SOL_SOCKET, SO_RCVBUF, + (const void *) &ls->rcvbuf, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(SO_RCVBUF, %d) failed", + ls->rcvbuf); + goto failed; + } + } + + if (ls->sndbuf != -1) { + if (setsockopt(s, SOL_SOCKET, SO_SNDBUF, + (const void *) &ls->sndbuf, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(SO_SNDBUF, %d) failed", + ls->sndbuf); + goto failed; + } + } + +#if (NGX_HAVE_IP_PKTINFO) + if (ls->wildcard && ls->sockaddr->sa_family == AF_INET) { + if (setsockopt(s, IPPROTO_IP, IP_PKTINFO, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(IP_PKTINFO) failed"); + goto failed; + } + } +#endif + +#if (NGX_HAVE_INET6 && NGX_HAVE_IPV6_RECVPKTINFO) + if (ls->wildcard && ls->sockaddr->sa_family == AF_INET6) { + if (setsockopt(s, IPPROTO_IPV6, IPV6_RECVPKTINFO, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(IPV6_RECVPKTINFO) failed"); + goto failed; + } + } +#endif + +#if (NGX_HAVE_INET6 && defined IPV6_V6ONLY) + + if (ls->sockaddr->sa_family == AF_INET6) { + int ipv6only; + + ipv6only = ls->ipv6only; + + if (setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, + (const void *) &ipv6only, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(IPV6_V6ONLY) %V failed", + &ls->addr_text); + goto failed; + } + } + +#endif + +#if (NGX_HAVE_IP_MTU_DISCOVER) + + if (ls->sockaddr->sa_family == AF_INET) { + value = IP_PMTUDISC_DO; + + if (setsockopt(s, IPPROTO_IP, IP_MTU_DISCOVER, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(IP_MTU_DISCOVER) " + "failed, ignored"); + } + } + +#elif (NGX_HAVE_IP_DONTFRAG) + + if (ls->sockaddr->sa_family == AF_INET) { + value = 1; + + if (setsockopt(s, IPPROTO_IP, IP_DONTFRAG, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(IP_DONTFRAG) " + "failed, ignored"); + } + } + +#endif + +#if (NGX_HAVE_INET6) + +#if (NGX_HAVE_IPV6_MTU_DISCOVER) + + if (ls->sockaddr->sa_family == AF_INET6) { + value = IPV6_PMTUDISC_DO; + + if (setsockopt(s, IPPROTO_IPV6, IPV6_MTU_DISCOVER, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(IPV6_MTU_DISCOVER) " + "failed, ignored"); + } + } + +#elif (NGX_HAVE_IPV6_DONTFRAG) + + if (ls->sockaddr->sa_family == AF_INET6) { + value = 1; + + if (setsockopt(s, IPPROTO_IPV6, IPV6_DONTFRAG, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(IPV6_DONTFRAG) " + "failed, ignored"); + } + } + +#endif + +#endif + + if (bind(s, ls->sockaddr, ls->socklen) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf bind() failed"); + goto failed; + } + + if (RAND_bytes((u_char *) &key, sizeof(uint64_t)) != 1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, + "quic bpf RAND_bytes() failed"); + goto failed; + } + + if ((key >> 56) == 0xFF) { + key &= ~((uint64_t) 0xFF << 56); + } + + map_value = s; + if (ngx_quic_bpf_map_update(cycle->log, grp->sock_map, + &key, &map_value, "sockmap") + != NGX_OK) + { + goto failed; + } + + map_updated = 1; + + if (ls->worker >= grp->listening.nelts) { + n = ls->worker + 1 - grp->listening.nelts; + + bls = ngx_array_push_n(&grp->listening, n); + if (bls == NULL) { + goto failed; + } + + ngx_memzero(bls, n * sizeof(ngx_quic_bpf_listening_t)); + + for (i = 0; i < n; i++) { + bls[i].fd = (ngx_socket_t) -1; + } + } + + bls = grp->listening.elts; + bls[ls->worker].fd = s; + bls[ls->worker].key = key; + bls[ls->worker].listening = ls; + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + "quic bpf socket fd:%d key:%uL", s, key); + + return NGX_OK; + +failed: + + if (map_updated) { + if (ngx_bpf_map_delete(grp->sock_map, &key) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "quic bpf failed to delete map entry"); + } + } + + if (ngx_close_socket(s) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf " ngx_close_socket_n " failed"); + } + + return NGX_ERROR; } @@ -491,31 +971,21 @@ ngx_quic_bpf_export_maps(ngx_cycle_t *cycle) ccf = ngx_core_get_conf(cycle); bcf = ngx_quic_bpf_get_conf(cycle); + if (!bcf->enabled) { + return NGX_OK; + } + + if (ngx_queue_empty(&bcf->groups)) { + return NGX_OK; + } len = sizeof(NGX_QUIC_BPF_VARNAME) + 1; - q = ngx_queue_head(&bcf->groups); - - while (q != ngx_queue_sentinel(&bcf->groups)) { - - grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue); - - q = ngx_queue_next(q); - - if (grp->unused) { - /* - * map was inherited, but it is not used in this configuration; - * do not pass such map further and drop the group to prevent - * interference with changes during reload - */ - - ngx_quic_bpf_close(cycle->log, grp->map_fd, "map"); - ngx_queue_remove(&grp->queue); - - continue; - } - - len += NGX_INT32_LEN + 1 + NGX_SOCKADDR_STRLEN + 1; + for (q = ngx_queue_head(&bcf->groups); + q != ngx_queue_sentinel(&bcf->groups); + q = ngx_queue_next(q)) + { + len += (NGX_INT32_LEN + 1) * 2 + NGX_SOCKADDR_STRLEN + 1; } len++; @@ -528,14 +998,18 @@ ngx_quic_bpf_export_maps(ngx_cycle_t *cycle) p = ngx_cpymem(buf, NGX_QUIC_BPF_VARNAME "=", sizeof(NGX_QUIC_BPF_VARNAME)); + *p++ = bcf->master_index ? '-' : '+'; + for (q = ngx_queue_head(&bcf->groups); q != ngx_queue_sentinel(&bcf->groups); q = ngx_queue_next(q)) { grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue); - p = ngx_sprintf(p, "%ud", grp->map_fd); + p = ngx_sprintf(p, "%ud", grp->sock_map); + *p++ = NGX_QUIC_BPF_ADDRSEP; + p = ngx_sprintf(p, "%ud", grp->worker_counts_map); *p++ = NGX_QUIC_BPF_ADDRSEP; p += ngx_sock_ntop(grp->sockaddr, grp->socklen, p, @@ -561,90 +1035,101 @@ ngx_quic_bpf_export_maps(ngx_cycle_t *cycle) static ngx_int_t ngx_quic_bpf_import_maps(ngx_cycle_t *cycle) { - int s; + int fds[2]; u_char *inherited, *p, *v; - ngx_uint_t in_fd; + ngx_int_t fd; + ngx_uint_t i, nfd; ngx_addr_t tmp; ngx_quic_bpf_conf_t *bcf; ngx_quic_sock_group_t *grp; - inherited = (u_char *) getenv(NGX_QUIC_BPF_VARNAME); + bcf = ngx_quic_bpf_get_conf(cycle); + inherited = bcf->env; if (inherited == NULL) { return NGX_OK; } - bcf = ngx_quic_bpf_get_conf(cycle); + nfd = 0; -#if (NGX_SUPPRESS_WARN) - s = -1; -#endif + ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, + "quic bpf using inherited QUIC BPF maps from \"%s\"", + inherited); - in_fd = 1; + switch (*inherited) { - for (p = inherited, v = p; *p; p++) { + case '+': + bcf->master_index = 1; + break; + + case '-': + bcf->master_index = 0; + break; + + default: + goto failed; + } + + for (p = inherited + 1, v = p; *p; p++) { switch (*p) { case NGX_QUIC_BPF_ADDRSEP: - if (!in_fd) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, - "quic bpf failed to parse inherited env"); - return NGX_ERROR; - } - in_fd = 0; - - s = ngx_atoi(v, p - v); - if (s == NGX_ERROR) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, - "quic bpf failed to parse inherited map fd"); - return NGX_ERROR; + if (nfd > 1) { + goto failed; } + fd = ngx_atoi(v, p - v); + if (fd == NGX_ERROR) { + goto failed; + } + + fds[nfd++] = fd; v = p + 1; break; case NGX_QUIC_BPF_VARSEP: - if (in_fd) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, - "quic bpf failed to parse inherited env"); - return NGX_ERROR; + if (nfd != 2) { + goto failed; } - in_fd = 1; - grp = ngx_pcalloc(cycle->pool, - sizeof(ngx_quic_sock_group_t)); + grp = ngx_pcalloc(cycle->pool, sizeof(ngx_quic_sock_group_t)); if (grp == NULL) { - return NGX_ERROR; + goto failed; } - grp->map_fd = s; - - if (ngx_parse_addr_port(cycle->pool, &tmp, v, p - v) + if (ngx_array_init(&grp->listening, cycle->pool, 1, + sizeof(ngx_quic_bpf_listening_t)) != NGX_OK) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, - "quic bpf failed to parse inherited" - " address '%*s'", p - v , v); - - ngx_quic_bpf_close(cycle->log, s, "inherited map"); - - return NGX_ERROR; + goto failed; } - grp->sockaddr = tmp.sockaddr; - grp->socklen = tmp.socklen; + grp->sock_map = fds[0]; + grp->worker_counts_map = fds[1]; - grp->unused = 1; + if (ngx_parse_addr_port(cycle->pool, &tmp, v, p - v) != NGX_OK) { + goto failed; + } + + grp->sockaddr = ngx_pcalloc(cycle->pool, tmp.socklen); + if (grp->sockaddr == NULL) { + goto failed; + } + + ngx_memcpy(grp->sockaddr, tmp.sockaddr, tmp.socklen); + grp->socklen = tmp.socklen; ngx_queue_insert_tail(&bcf->groups, &grp->queue); - ngx_log_debug3(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + ngx_log_debug4(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "quic bpf sockmap inherited with " - "fd:%d address:%*s", - grp->map_fd, p - v, v); + "fds:%d/%d address:%*s", + fds[0], fds[1], p - v, v); + + nfd = 0; v = p + 1; break; @@ -653,5 +1138,120 @@ ngx_quic_bpf_import_maps(ngx_cycle_t *cycle) } } + return NGX_OK; + +failed: + + for (i = 0; i < nfd; i++) { + ngx_quic_bpf_close(cycle->log, fds[i], "inherited"); + } + + ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, + "quic bpf failed to parse inherited QUIC BPF variable"); + + return NGX_ERROR; +} + + +ngx_int_t +ngx_quic_bpf_get_worker_fd(ngx_connection_t *lc, ngx_socket_t *fd) +{ + ngx_event_t *rev; + ngx_connection_t *c; + ngx_quic_bpf_conf_t *bcf; + ngx_quic_sock_group_t *grp; + ngx_quic_bpf_listening_t *bpf_listening, *bls; + + bcf = ngx_quic_bpf_get_conf((ngx_cycle_t *) ngx_cycle); + + if (!bcf->enabled || !lc->listening->quic || !lc->listening->reuseport) { + *fd = lc->fd; + return NGX_OK; + } + + grp = ngx_quic_bpf_find_group(bcf, lc->listening); + + if (grp == NULL) { + *fd = lc->fd; + return NGX_OK; + } + + if (ngx_worker >= grp->listening.nelts) { + return NGX_ERROR; + } + + bpf_listening = grp->listening.elts; + bls = &bpf_listening[ngx_worker]; + + if (bls->fd == (ngx_socket_t) -1) { + return NGX_ERROR; + } + + if (bls->connection == NULL) { + c = ngx_get_connection(bls->fd, lc->log); + if (c == NULL) { + return NGX_ERROR; + } + + c->type = SOCK_DGRAM; + c->log = lc->log; + c->listening = bls->listening; + + rev = c->read; + rev->quic = 1; + rev->log = c->log; + rev->handler = ngx_quic_recvmsg; + + if (ngx_add_event(rev, NGX_READ_EVENT, 0) == NGX_ERROR) { + ngx_free_connection(c); + return NGX_ERROR; + } + + bls->connection = c; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, lc->log, 0, + "quic bpf socket connection fd:%d", bls->fd); + } + + *fd = bls->fd; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, lc->log, 0, + "quic bpf socket fd:%d", bls->fd); + + return NGX_OK; +} + + +ngx_int_t +ngx_quic_bpf_get_worker_key(ngx_connection_t *c, uint64_t *key) +{ + ngx_quic_bpf_conf_t *bcf; + ngx_quic_sock_group_t *grp; + ngx_quic_bpf_listening_t *bls; + + bcf = ngx_quic_bpf_get_conf((ngx_cycle_t *) ngx_cycle); + + if (!bcf->enabled || !c->listening->quic || !c->listening->reuseport) { + return NGX_DECLINED; + } + + grp = ngx_quic_bpf_find_group(bcf, c->listening); + + if (grp == NULL) { + return NGX_DECLINED; + } + + if (ngx_worker >= grp->listening.nelts) { + return NGX_ERROR; + } + + bls = grp->listening.elts; + + if (bls[ngx_worker].fd == (ngx_socket_t) -1) { + return NGX_ERROR; + } + + *key = bls[ngx_worker].key; + return NGX_OK; } diff --git a/src/event/quic/ngx_event_quic_bpf.h b/src/event/quic/ngx_event_quic_bpf.h new file mode 100644 index 000000000..eaac8fb4f --- /dev/null +++ b/src/event/quic/ngx_event_quic_bpf.h @@ -0,0 +1,20 @@ + +/* + * Copyright (C) Nginx, Inc. + */ + + +#ifndef _NGX_EVENT_QUIC_BPF_H_INCLUDED_ +#define _NGX_EVENT_QUIC_BPF_H_INCLUDED_ + + +#include +#include + + +ngx_int_t ngx_quic_bpf_get_worker_fd(ngx_connection_t *lc, + ngx_socket_t *fd); +ngx_int_t ngx_quic_bpf_get_worker_key(ngx_connection_t *c, uint64_t *key); + + +#endif /* _NGX_EVENT_QUIC_BPF_H_INCLUDED_ */ diff --git a/src/event/quic/ngx_event_quic_bpf_code.c b/src/event/quic/ngx_event_quic_bpf_code.c index e32f1c68e..2461cac43 100644 --- a/src/event/quic/ngx_event_quic_bpf_code.c +++ b/src/event/quic/ngx_event_quic_bpf_code.c @@ -7,93 +7,194 @@ static ngx_bpf_reloc_t bpf_reloc_prog_ngx_quic_reuseport_helper[] = { - { "ngx_quic_sockmap", 77 }, + { "ngx_quic_sockmap", 81 }, + { "ngx_quic_worker_counts", 97 }, + { "ngx_quic_worker_counts", 106 }, + { "ngx_quic_sockmap", 133 }, + { "ngx_quic_sockmap", 170 }, }; static struct bpf_insn bpf_insn_prog_ngx_quic_reuseport_helper[] = { /* opcode dst src offset imm */ { 0xbf, BPF_REG_6, BPF_REG_1, (int16_t) 0, 0x0 }, - { 0x79, BPF_REG_8, BPF_REG_6, (int16_t) 8, 0x0 }, - { 0x79, BPF_REG_7, BPF_REG_6, (int16_t) 0, 0x0 }, - { 0xbf, BPF_REG_1, BPF_REG_7, (int16_t) 0, 0x0 }, + { 0x79, BPF_REG_9, BPF_REG_6, (int16_t) 8, 0x0 }, + { 0x79, BPF_REG_8, BPF_REG_6, (int16_t) 0, 0x0 }, + { 0xbf, BPF_REG_1, BPF_REG_8, (int16_t) 0, 0x0 }, { 0x7, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x8 }, - { 0x2d, BPF_REG_8, BPF_REG_1, (int16_t) 10, 0x0 }, + { 0x2d, BPF_REG_9, BPF_REG_1, (int16_t) 11, 0x0 }, { 0xbf, BPF_REG_3, BPF_REG_10, (int16_t) 0, 0x0 }, - { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xfffffff7 }, + { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xffffffef }, { 0xbf, BPF_REG_1, BPF_REG_6, (int16_t) 0, 0x0 }, { 0xb7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x8 }, { 0xb7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x1 }, { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x1a }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 }, { 0x15, BPF_REG_0, BPF_REG_0, (int16_t) 1, 0x0 }, - { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 67, 0x0 }, - { 0x71, BPF_REG_1, BPF_REG_10, (int16_t) 65527, 0x0 }, + { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 163, 0x0 }, + { 0x71, BPF_REG_1, BPF_REG_10, (int16_t) 65519, 0x0 }, { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 2, 0x0 }, { 0x71, BPF_REG_1, BPF_REG_1, (int16_t) 0, 0x0 }, - { 0x73, BPF_REG_10, BPF_REG_1, (int16_t) 65527, 0x0 }, + { 0x73, BPF_REG_10, BPF_REG_1, (int16_t) 65519, 0x0 }, { 0xb7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x9 }, { 0x67, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x38 }, { 0xc7, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x38 }, { 0x65, BPF_REG_1, BPF_REG_0, (int16_t) 18, 0xffffffff }, - { 0xbf, BPF_REG_1, BPF_REG_7, (int16_t) 0, 0x0 }, + { 0xbf, BPF_REG_1, BPF_REG_8, (int16_t) 0, 0x0 }, { 0x7, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0xd }, - { 0x2d, BPF_REG_8, BPF_REG_1, (int16_t) 11, 0x0 }, + { 0x2d, BPF_REG_9, BPF_REG_1, (int16_t) 11, 0x0 }, { 0xbf, BPF_REG_3, BPF_REG_10, (int16_t) 0, 0x0 }, - { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xfffffff7 }, + { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xffffffef }, { 0xbf, BPF_REG_1, BPF_REG_6, (int16_t) 0, 0x0 }, { 0xb7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xd }, { 0xb7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x1 }, { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x1a }, - { 0x55, BPF_REG_0, BPF_REG_0, (int16_t) 49, 0x0 }, - { 0xb7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xe }, - { 0x71, BPF_REG_1, BPF_REG_10, (int16_t) 65527, 0x0 }, - { 0x15, BPF_REG_1, BPF_REG_0, (int16_t) 5, 0x14 }, - { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 45, 0x0 }, - { 0xb7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xe }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x15, BPF_REG_0, BPF_REG_0, (int16_t) 1, 0x0 }, + { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 143, 0x0 }, + { 0x71, BPF_REG_1, BPF_REG_10, (int16_t) 65519, 0x0 }, + { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 2, 0x0 }, { 0x71, BPF_REG_1, BPF_REG_1, (int16_t) 0, 0x0 }, - { 0x73, BPF_REG_10, BPF_REG_1, (int16_t) 65527, 0x0 }, - { 0x55, BPF_REG_1, BPF_REG_0, (int16_t) 41, 0x14 }, - { 0xf, BPF_REG_7, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0xbf, BPF_REG_1, BPF_REG_7, (int16_t) 0, 0x0 }, + { 0x73, BPF_REG_10, BPF_REG_1, (int16_t) 65519, 0x0 }, + { 0xb7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xe }, + { 0x55, BPF_REG_1, BPF_REG_0, (int16_t) 52, 0x14 }, + { 0xf, BPF_REG_8, BPF_REG_2, (int16_t) 0, 0x0 }, + { 0xbf, BPF_REG_1, BPF_REG_8, (int16_t) 0, 0x0 }, { 0x7, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x14 }, - { 0x3d, BPF_REG_8, BPF_REG_1, (int16_t) 7, 0x0 }, - { 0xbf, BPF_REG_7, BPF_REG_10, (int16_t) 0, 0x0 }, - { 0x7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0xffffffe3 }, + { 0x3d, BPF_REG_9, BPF_REG_1, (int16_t) 8, 0x0 }, + { 0xbf, BPF_REG_8, BPF_REG_10, (int16_t) 0, 0x0 }, + { 0x7, BPF_REG_8, BPF_REG_0, (int16_t) 0, 0xffffffdb }, { 0xbf, BPF_REG_1, BPF_REG_6, (int16_t) 0, 0x0 }, - { 0xbf, BPF_REG_3, BPF_REG_7, (int16_t) 0, 0x0 }, + { 0xbf, BPF_REG_3, BPF_REG_8, (int16_t) 0, 0x0 }, { 0xb7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x14 }, { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x1a }, - { 0x55, BPF_REG_0, BPF_REG_0, (int16_t) 30, 0x0 }, - { 0x71, BPF_REG_2, BPF_REG_7, (int16_t) 0, 0x0 }, - { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x38 }, - { 0x71, BPF_REG_1, BPF_REG_7, (int16_t) 1, 0x0 }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x55, BPF_REG_0, BPF_REG_0, (int16_t) 125, 0x0 }, + { 0x71, BPF_REG_1, BPF_REG_8, (int16_t) 1, 0x0 }, { 0x67, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x30 }, - { 0x4f, BPF_REG_1, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0x71, BPF_REG_2, BPF_REG_7, (int16_t) 2, 0x0 }, - { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x28 }, - { 0x4f, BPF_REG_1, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0x71, BPF_REG_2, BPF_REG_7, (int16_t) 3, 0x0 }, - { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x20 }, - { 0x4f, BPF_REG_1, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0x71, BPF_REG_2, BPF_REG_7, (int16_t) 4, 0x0 }, - { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x18 }, - { 0x4f, BPF_REG_1, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0x71, BPF_REG_2, BPF_REG_7, (int16_t) 5, 0x0 }, - { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x10 }, - { 0x4f, BPF_REG_1, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0x71, BPF_REG_2, BPF_REG_7, (int16_t) 6, 0x0 }, - { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x8 }, - { 0x4f, BPF_REG_1, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0x71, BPF_REG_2, BPF_REG_7, (int16_t) 7, 0x0 }, - { 0x4f, BPF_REG_1, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0x7b, BPF_REG_10, BPF_REG_1, (int16_t) 65528, 0x0 }, + { 0x71, BPF_REG_2, BPF_REG_8, (int16_t) 0, 0x0 }, + { 0xbf, BPF_REG_3, BPF_REG_2, (int16_t) 0, 0x0 }, + { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x38 }, + { 0x4f, BPF_REG_1, BPF_REG_3, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_3, BPF_REG_8, (int16_t) 2, 0x0 }, + { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x28 }, + { 0x4f, BPF_REG_1, BPF_REG_3, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_3, BPF_REG_8, (int16_t) 3, 0x0 }, + { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x4f, BPF_REG_1, BPF_REG_3, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_3, BPF_REG_8, (int16_t) 4, 0x0 }, + { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x18 }, + { 0x4f, BPF_REG_1, BPF_REG_3, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_3, BPF_REG_8, (int16_t) 5, 0x0 }, + { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x10 }, + { 0x4f, BPF_REG_1, BPF_REG_3, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_3, BPF_REG_8, (int16_t) 7, 0x0 }, + { 0x4f, BPF_REG_1, BPF_REG_3, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_3, BPF_REG_8, (int16_t) 6, 0x0 }, + { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x8 }, + { 0x4f, BPF_REG_1, BPF_REG_3, (int16_t) 0, 0x0 }, + { 0x7b, BPF_REG_10, BPF_REG_1, (int16_t) 65520, 0x0 }, + { 0x15, BPF_REG_2, BPF_REG_0, (int16_t) 15, 0xff }, { 0xbf, BPF_REG_3, BPF_REG_10, (int16_t) 0, 0x0 }, - { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xfffffff8 }, + { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xfffffff0 }, { 0xbf, BPF_REG_1, BPF_REG_6, (int16_t) 0, 0x0 }, { 0x18, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x0 }, { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, { 0xb7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x0 }, { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x52 }, - { 0xb7, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x1 }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x1 }, + { 0x67, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x77, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x15, BPF_REG_0, BPF_REG_0, (int16_t) 89, 0x0 }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0xfffffffe }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x1d, BPF_REG_0, BPF_REG_1, (int16_t) 1, 0x0 }, + { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 84, 0x0 }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x63, BPF_REG_10, BPF_REG_7, (int16_t) 65532, 0x0 }, + { 0xbf, BPF_REG_2, BPF_REG_10, (int16_t) 0, 0x0 }, + { 0x7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xfffffffc }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x1 }, + { 0xbf, BPF_REG_8, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x15, BPF_REG_8, BPF_REG_0, (int16_t) 76, 0x0 }, + { 0xb7, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x1 }, + { 0x63, BPF_REG_10, BPF_REG_1, (int16_t) 65532, 0x0 }, + { 0xbf, BPF_REG_2, BPF_REG_10, (int16_t) 0, 0x0 }, + { 0x7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xfffffffc }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x1 }, + { 0x15, BPF_REG_0, BPF_REG_0, (int16_t) 68, 0x0 }, + { 0x61, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x61, BPF_REG_9, BPF_REG_8, (int16_t) 0, 0x0 }, + { 0x2d, BPF_REG_9, BPF_REG_1, (int16_t) 1, 0x0 }, + { 0xbf, BPF_REG_9, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x15, BPF_REG_9, BPF_REG_0, (int16_t) 63, 0x0 }, + { 0xbf, BPF_REG_1, BPF_REG_9, (int16_t) 0, 0x0 }, + { 0x67, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x77, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0xbf, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x61, BPF_REG_2, BPF_REG_6, (int16_t) 32, 0x0 }, + { 0x9f, BPF_REG_2, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0xc7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0xffffffff }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0xffff }, + { 0x5f, BPF_REG_2, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0xff000000 }, + { 0x4f, BPF_REG_2, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x7b, BPF_REG_10, BPF_REG_2, (int16_t) 65520, 0x0 }, + { 0xbf, BPF_REG_3, BPF_REG_10, (int16_t) 0, 0x0 }, + { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xfffffff0 }, + { 0xbf, BPF_REG_1, BPF_REG_6, (int16_t) 0, 0x0 }, + { 0x18, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0xb7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x52 }, + { 0xbf, BPF_REG_2, BPF_REG_7, (int16_t) 0, 0x0 }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x1 }, + { 0x67, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x77, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x15, BPF_REG_0, BPF_REG_0, (int16_t) 36, 0x0 }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0xfffffffe }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x1d, BPF_REG_0, BPF_REG_1, (int16_t) 1, 0x0 }, + { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 31, 0x0 }, + { 0x61, BPF_REG_1, BPF_REG_8, (int16_t) 0, 0x0 }, + { 0x61, BPF_REG_2, BPF_REG_2, (int16_t) 0, 0x0 }, + { 0x2d, BPF_REG_2, BPF_REG_1, (int16_t) 1, 0x0 }, + { 0xbf, BPF_REG_1, BPF_REG_2, (int16_t) 0, 0x0 }, + { 0x15, BPF_REG_1, BPF_REG_0, (int16_t) 26, 0x0 }, + { 0x67, BPF_REG_9, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x77, BPF_REG_9, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x67, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x77, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x1d, BPF_REG_1, BPF_REG_9, (int16_t) 21, 0x0 }, + { 0x61, BPF_REG_2, BPF_REG_6, (int16_t) 32, 0x0 }, + { 0x9f, BPF_REG_2, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0xc7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0xffffffff }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0xffff }, + { 0x5f, BPF_REG_2, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0xff000000 }, + { 0x4f, BPF_REG_2, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x7b, BPF_REG_10, BPF_REG_2, (int16_t) 65520, 0x0 }, + { 0xbf, BPF_REG_3, BPF_REG_10, (int16_t) 0, 0x0 }, + { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xfffffff0 }, + { 0xbf, BPF_REG_1, BPF_REG_6, (int16_t) 0, 0x0 }, + { 0x18, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0xb7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x52 }, + { 0x67, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x77, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x15, BPF_REG_0, BPF_REG_0, (int16_t) 1, 0x0 }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0xbf, BPF_REG_0, BPF_REG_7, (int16_t) 0, 0x0 }, { 0x95, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, }; diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h index efcb632ca..40873cf1e 100644 --- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -48,6 +48,9 @@ typedef struct ngx_quic_keys_s ngx_quic_keys_t; #include #include #include +#if (NGX_QUIC_BPF) +#include +#endif /* RFC 9002, 6.2.2. Handshakes and New Paths: kInitialRtt */ diff --git a/src/event/quic/ngx_event_quic_connid.c b/src/event/quic/ngx_event_quic_connid.c index 4e7b8dc22..b76eaaa5f 100644 --- a/src/event/quic/ngx_event_quic_connid.c +++ b/src/event/quic/ngx_event_quic_connid.c @@ -34,7 +34,7 @@ ngx_quic_create_server_id(ngx_connection_t *c, u_char *id) if (ngx_quic_bpf_attach_id(c, id) != NGX_OK) { ngx_log_error(NGX_LOG_ERR, c->log, 0, "quic bpf failed to generate socket key"); - /* ignore error, things still may work */ + return NGX_ERROR; } #endif @@ -47,22 +47,21 @@ ngx_quic_create_server_id(ngx_connection_t *c, u_char *id) static ngx_int_t ngx_quic_bpf_attach_id(ngx_connection_t *c, u_char *id) { - int fd; - uint64_t cookie; - socklen_t optlen; + uint64_t key; + ngx_int_t rc; - fd = c->listening->fd; + rc = ngx_quic_bpf_get_worker_key(c, &key); - optlen = sizeof(cookie); - - if (getsockopt(fd, SOL_SOCKET, SO_COOKIE, &cookie, &optlen) == -1) { - ngx_log_error(NGX_LOG_ERR, c->log, ngx_socket_errno, - "quic getsockopt(SO_COOKIE) failed"); + if (rc == NGX_DECLINED) { + /* BPF not applicable for this listener, leave DCID as random */ + return NGX_OK; + } + if (rc != NGX_OK) { return NGX_ERROR; } - ngx_quic_dcid_encode_key(id, cookie); + ngx_quic_dcid_encode_key(id, key); return NGX_OK; } diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index f98c834a1..cb75d2868 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -1085,7 +1085,6 @@ ngx_quic_send_retry(ngx_connection_t *c, ngx_quic_conf_t *conf, pkt.odcid = inpkt->dcid; pkt.dcid = inpkt->scid; - /* TODO: generate routable dcid */ if (RAND_bytes(dcid, NGX_QUIC_SERVER_CID_LEN) != 1) { return NGX_ERROR; } diff --git a/src/event/quic/ngx_event_quic_udp.c b/src/event/quic/ngx_event_quic_udp.c index 15b54bc82..46c741ce6 100644 --- a/src/event/quic/ngx_event_quic_udp.c +++ b/src/event/quic/ngx_event_quic_udp.c @@ -26,6 +26,7 @@ ngx_quic_recvmsg(ngx_event_t *ev) ngx_err_t err; socklen_t socklen, local_socklen; ngx_event_t *rev, *wev; + ngx_socket_t cfd; struct iovec iov[1]; struct msghdr msg; ngx_sockaddr_t sa, lsa; @@ -168,7 +169,7 @@ ngx_quic_recvmsg(ngx_event_t *ev) c->log->handler = NULL; ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic recvmsg: fd:%d n:%z", c->fd, n); + "quic recvmsg: fd:%d n:%z", lc->fd, n); c->log->handler = handler; } @@ -204,6 +205,14 @@ ngx_quic_recvmsg(ngx_event_t *ev) goto next; } +#if (NGX_QUIC_BPF) + if (lc->read->quic) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, + "quic new connection on worker socket, dropped"); + goto next; + } +#endif + #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_accepted, 1); #endif @@ -211,7 +220,15 @@ ngx_quic_recvmsg(ngx_event_t *ev) ngx_accept_disabled = ngx_cycle->connection_n / 8 - ngx_cycle->free_connection_n; - c = ngx_get_connection(lc->fd, ev->log); + cfd = lc->fd; + +#if (NGX_QUIC_BPF) + if (ngx_quic_bpf_get_worker_fd(lc, &cfd) != NGX_OK) { + return; + } +#endif + + c = ngx_get_connection(cfd, ev->log); if (c == NULL) { return; } @@ -327,7 +344,7 @@ ngx_quic_recvmsg(ngx_event_t *ev) ngx_log_debug4(NGX_LOG_DEBUG_EVENT, log, 0, "*%uA quic recvmsg: %V fd:%d n:%z", - c->number, &addr, c->fd, n); + c->number, &addr, lc->fd, n); } } diff --git a/src/os/unix/ngx_process_cycle.c b/src/os/unix/ngx_process_cycle.c index 22a0e07f2..ede5ae0ba 100644 --- a/src/os/unix/ngx_process_cycle.c +++ b/src/os/unix/ngx_process_cycle.c @@ -989,7 +989,8 @@ ngx_worker_process_exit(ngx_cycle_t *cycle) && c[i].read && !c[i].read->accept && !c[i].read->channel - && !c[i].read->resolver) + && !c[i].read->resolver + && !c[i].read->quic) { ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "*%uA open socket #%d left in connection %ui",