diff --git a/auto/modules b/auto/modules index 0f037b278..99e7b4f3b 100644 --- a/auto/modules +++ b/auto/modules @@ -1455,11 +1455,11 @@ if [ $USE_OPENSSL_QUIC = YES ]; then . auto/module - if [ $QUIC_BPF = YES -a $SO_COOKIE_FOUND = YES ]; then + if [ $QUIC_BPF = YES ]; then ngx_module_type=CORE ngx_module_name=ngx_quic_bpf_module ngx_module_incs= - ngx_module_deps= + ngx_module_deps=src/event/quic/ngx_event_quic_bpf.h ngx_module_srcs="src/event/quic/ngx_event_quic_bpf.c \ src/event/quic/ngx_event_quic_bpf_code.c" ngx_module_libs= diff --git a/auto/options b/auto/options index 332d74efb..7220174a9 100644 --- a/auto/options +++ b/auto/options @@ -178,8 +178,6 @@ USE_GEOIP=NO NGX_GOOGLE_PERFTOOLS=NO NGX_CPP_TEST=NO -SO_COOKIE_FOUND=NO - NGX_LIBATOMIC=NO NGX_CPU_CACHE_LINE= diff --git a/auto/os/linux b/auto/os/linux index bc0556b3a..df8d03451 100644 --- a/auto/os/linux +++ b/auto/os/linux @@ -249,7 +249,10 @@ ngx_feature_test="union bpf_attr attr = { 0 }; attr.map_flags = 0; attr.map_type = BPF_MAP_TYPE_SOCKHASH; + syscall(__NR_bpf, 0, &attr, 0); + attr.map_flags = 0; + attr.map_type = BPF_MAP_TYPE_ARRAY; syscall(__NR_bpf, 0, &attr, 0);" . auto/feature @@ -263,23 +266,6 @@ if [ $ngx_found = yes ]; then fi -ngx_feature="SO_COOKIE" -ngx_feature_name="NGX_HAVE_SO_COOKIE" -ngx_feature_run=no -ngx_feature_incs="#include - $NGX_INCLUDE_INTTYPES_H" -ngx_feature_path= -ngx_feature_libs= -ngx_feature_test="socklen_t optlen = sizeof(uint64_t); - uint64_t cookie; - getsockopt(0, SOL_SOCKET, SO_COOKIE, &cookie, &optlen)" -. auto/feature - -if [ $ngx_found = yes ]; then - SO_COOKIE_FOUND=YES -fi - - # UDP segmentation offloading ngx_feature="UDP_SEGMENT" diff --git a/src/core/ngx_bpf.c b/src/core/ngx_bpf.c index 363a02c7d..d08b7c41c 100644 --- a/src/core/ngx_bpf.c +++ b/src/core/ngx_bpf.c @@ -7,7 +7,7 @@ #include #include -#define NGX_BPF_LOGBUF_SIZE (16 * 1024) +#define NGX_BPF_LOGBUF_SIZE (32 * 1024) static ngx_inline int diff --git a/src/core/ngx_connection.c b/src/core/ngx_connection.c index 7ea7706e5..faf987bbf 100644 --- a/src/core/ngx_connection.c +++ b/src/core/ngx_connection.c @@ -1142,7 +1142,9 @@ ngx_close_listening_sockets(ngx_cycle_t *cycle) for (i = 0; i < cycle->listening.nelts; i++) { #if (NGX_QUIC) - if (ls[i].quic) { + if (ls[i].quic + && (!ls[i].reuseport || !ngx_quic_bpf_enabled(cycle))) + { continue; } #endif diff --git a/src/event/ngx_event.h b/src/event/ngx_event.h index deac04e7b..5196353bc 100644 --- a/src/event/ngx_event.h +++ b/src/event/ngx_event.h @@ -73,6 +73,7 @@ struct ngx_event_s { /* to test on worker exit */ unsigned channel:1; unsigned resolver:1; + unsigned quic:1; unsigned cancelable:1; diff --git a/src/event/quic/bpf/makefile b/src/event/quic/bpf/makefile index 277aa3151..e8e449fc8 100644 --- a/src/event/quic/bpf/makefile +++ b/src/event/quic/bpf/makefile @@ -1,4 +1,7 @@ -CFLAGS=-O2 -Wall + +# Copyright (C) Nginx, Inc. + +CFLAGS=-O2 -Wall $(MAKE_CFLAGS) LICENSE=BSD diff --git a/src/event/quic/bpf/ngx_quic_reuseport_helper.c b/src/event/quic/bpf/ngx_quic_reuseport_helper.c index cf7a24c5a..51f9923a5 100644 --- a/src/event/quic/bpf/ngx_quic_reuseport_helper.c +++ b/src/event/quic/bpf/ngx_quic_reuseport_helper.c @@ -54,17 +54,23 @@ char _license[] SEC("license") = LICENSE; ((__u64)(p)[6] << 8) | \ ((__u64)(p)[7])) +#define ngx_quic_bpf_listen_key(worker) \ + (((__u64) 0xFF << 56) | ((__u64) (worker) & 0xFFFFFFFFFFFFULL)) + + /* * actual map object is created by the "bpf" system call, * all pointers to this variable are replaced by the bpf loader */ -extern int ngx_quic_sockmap; +struct {} ngx_quic_sockmap SEC(".maps"); +struct {} ngx_quic_worker_counts SEC(".maps"); SEC(PROGNAME) int ngx_quic_select_socket_by_dcid(struct sk_reuseport_md *ctx) { - int rc; + int rc, worker_idx; + __u32 wc_key, *wc0, *wc1, n, m; __u64 key; size_t offset; unsigned char *start, *end, *dcid, byte, buf[NGX_QUIC_SERVER_CID_LEN]; @@ -98,7 +104,7 @@ int ngx_quic_select_socket_by_dcid(struct sk_reuseport_md *ctx) } if (byte != NGX_QUIC_SERVER_CID_LEN) { - return SK_PASS; + goto new; } } @@ -118,30 +124,84 @@ int ngx_quic_select_socket_by_dcid(struct sk_reuseport_md *ctx) key = ngx_quic_parse_uint64(dcid); - rc = bpf_sk_select_reuseport(ctx, &ngx_quic_sockmap, &key, 0); - - switch (rc) { - case 0: - debugmsg("nginx quic socket selected by key 0x%llx", key); - return SK_PASS; - - /* kernel returns positive error numbers, errno.h defines positive */ - case -ENOENT: - debugmsg("nginx quic default route for key 0x%llx", key); - /* let the default reuseport logic decide which socket to choose */ - return SK_PASS; - - default: - debugmsg("nginx quic bpf_sk_select_reuseport err: %d key 0x%llx", - rc, key); - goto failed; + if ((key >> 56) == 0xFF) { + goto new; } + rc = bpf_sk_select_reuseport(ctx, &ngx_quic_sockmap, &key, 0); + + if (rc == 0) { + debugmsg("nginx quic worker socket selected by dcid"); + return SK_PASS; + } + + if (rc != -ENOENT) { + debugmsg("nginx quic bpf_sk_select_reuseport() failed: %d", rc); + return SK_DROP; + } + +new: + + wc_key = 0; + wc0 = bpf_map_lookup_elem(&ngx_quic_worker_counts, &wc_key); + if (wc0 == NULL) { + debugmsg("nginx quic worker count 0 undefined"); + return SK_DROP; + } + + wc_key = 1; + wc1 = bpf_map_lookup_elem(&ngx_quic_worker_counts, &wc_key); + if (wc1 == NULL) { + debugmsg("nginx quic worker count 1 undefined"); + return SK_DROP; + } + + n = *wc0 > *wc1 ? *wc0 : *wc1; + + if (n == 0) { + debugmsg("nginx quic no active workers"); + return SK_DROP; + } + + worker_idx = ctx->hash % n; + key = ngx_quic_bpf_listen_key(worker_idx); + + rc = bpf_sk_select_reuseport(ctx, &ngx_quic_sockmap, &key, 0); + + if (rc == 0) { + debugmsg("nginx quic listener socket selected worker index:%d", + worker_idx); + return SK_PASS; + } + + if (rc != -ENOENT) { + debugmsg("nginx quic bpf_sk_select_reuseport() failed: %d", rc); + return SK_DROP; + } + + m = *wc0 < *wc1 ? *wc0 : *wc1; + + if (m && m != n) { + worker_idx = ctx->hash % m; + key = ngx_quic_bpf_listen_key(worker_idx); + + rc = bpf_sk_select_reuseport(ctx, &ngx_quic_sockmap, &key, 0); + + if (rc == 0) { + debugmsg("nginx quic listener socket selected " + "worker index:%d (fallback)", worker_idx); + return SK_PASS; + } + + debugmsg("nginx quic bpf_sk_select_reuseport() fallback " + "failed: %d", rc); + } + + return SK_DROP; + failed: - /* - * SK_DROP will generate ICMP, but we may want to process "invalid" packet - * in userspace quic to investigate further and finally react properly - * (maybe ignore, maybe send something in response or close connection) - */ - return SK_PASS; + + debugmsg("nginx quic bad datagram"); + + return SK_DROP; } diff --git a/src/event/quic/ngx_event_quic.h b/src/event/quic/ngx_event_quic.h index 4f899ec0a..e26673ac6 100644 --- a/src/event/quic/ngx_event_quic.h +++ b/src/event/quic/ngx_event_quic.h @@ -142,4 +142,10 @@ ngx_int_t ngx_quic_get_packet_dcid(ngx_log_t *log, u_char *data, size_t len, ngx_int_t ngx_quic_derive_key(ngx_log_t *log, const char *label, ngx_str_t *secret, ngx_str_t *salt, u_char *out, size_t len); +#if (NGX_QUIC_BPF) +ngx_uint_t ngx_quic_bpf_enabled(ngx_cycle_t *cycle); +#else +#define ngx_quic_bpf_enabled(c) 0 +#endif + #endif /* _NGX_EVENT_QUIC_H_INCLUDED_ */ diff --git a/src/event/quic/ngx_event_quic_bpf.c b/src/event/quic/ngx_event_quic_bpf.c index ab024ad56..2488d9c75 100644 --- a/src/event/quic/ngx_event_quic_bpf.c +++ b/src/event/quic/ngx_event_quic_bpf.c @@ -6,6 +6,8 @@ #include #include +#include +#include #define NGX_QUIC_BPF_VARNAME "NGINX_BPF_MAPS" @@ -23,42 +25,67 @@ #define ngx_core_get_conf(cycle) \ (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module) +#define ngx_quic_bpf_listen_key(worker) \ + (((uint64_t) 0xFF << 56) \ + | ((uint64_t) (worker) & 0xFFFFFFFFFFFFULL)) + typedef struct { ngx_queue_t queue; - int map_fd; + + int sock_map; + int worker_counts_map; struct sockaddr *sockaddr; socklen_t socklen; - ngx_uint_t unused; /* unsigned unused:1; */ + + ngx_array_t listening; } ngx_quic_sock_group_t; +typedef struct { + ngx_socket_t fd; + uint64_t key; + ngx_listening_t *listening; + ngx_connection_t *connection; +} ngx_quic_bpf_listening_t; + + typedef struct { ngx_flag_t enabled; ngx_uint_t map_size; + ngx_uint_t master_index; + u_char *env; ngx_queue_t groups; /* of ngx_quic_sock_group_t */ } ngx_quic_bpf_conf_t; static void *ngx_quic_bpf_create_conf(ngx_cycle_t *cycle); +static char *ngx_quic_bpf_init_conf(ngx_cycle_t *cycle, void *conf); static ngx_int_t ngx_quic_bpf_module_init(ngx_cycle_t *cycle); +static void ngx_quic_bpf_exit_master(ngx_cycle_t *cycle); static void ngx_quic_bpf_cleanup(void *data); static ngx_inline void ngx_quic_bpf_close(ngx_log_t *log, int fd, const char *name); +static ngx_inline ngx_int_t ngx_quic_bpf_map_update(ngx_log_t *log, int fd, + const void *key, const void *value, const char *name); static ngx_quic_sock_group_t *ngx_quic_bpf_find_group(ngx_quic_bpf_conf_t *bcf, ngx_listening_t *ls); static ngx_quic_sock_group_t *ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle, - struct sockaddr *sa, socklen_t socklen); + ngx_listening_t *ls); static ngx_quic_sock_group_t *ngx_quic_bpf_create_group(ngx_cycle_t *cycle, ngx_listening_t *ls); +static ngx_int_t ngx_quic_bpf_inherit_fd(ngx_cycle_t *cycle, int fd); static ngx_quic_sock_group_t *ngx_quic_bpf_get_group(ngx_cycle_t *cycle, ngx_listening_t *ls); static ngx_int_t ngx_quic_bpf_group_add_socket(ngx_cycle_t *cycle, ngx_listening_t *ls); -static uint64_t ngx_quic_bpf_socket_key(ngx_fd_t fd, ngx_log_t *log); +static ngx_int_t ngx_quic_bpf_add_worker_socket(ngx_cycle_t *cycle, + ngx_quic_sock_group_t *grp, ngx_listening_t *ls); +static ngx_int_t ngx_quic_bpf_publish_workers(ngx_cycle_t *cycle, + ngx_quic_sock_group_t *grp, ngx_uint_t worker_count); static ngx_int_t ngx_quic_bpf_export_maps(ngx_cycle_t *cycle); static ngx_int_t ngx_quic_bpf_import_maps(ngx_cycle_t *cycle); @@ -82,7 +109,7 @@ static ngx_command_t ngx_quic_bpf_commands[] = { static ngx_core_module_t ngx_quic_bpf_module_ctx = { ngx_string("quic_bpf"), ngx_quic_bpf_create_conf, - NULL + ngx_quic_bpf_init_conf }; @@ -97,7 +124,7 @@ ngx_module_t ngx_quic_bpf_module = { NULL, /* init thread */ NULL, /* exit thread */ NULL, /* exit process */ - NULL, /* exit master */ + ngx_quic_bpf_exit_master, /* exit master */ NGX_MODULE_V1_PADDING }; @@ -105,6 +132,8 @@ ngx_module_t ngx_quic_bpf_module = { static void * ngx_quic_bpf_create_conf(ngx_cycle_t *cycle) { + size_t len; + u_char *env; ngx_quic_bpf_conf_t *bcf; bcf = ngx_pcalloc(cycle->pool, sizeof(ngx_quic_bpf_conf_t)); @@ -115,20 +144,66 @@ ngx_quic_bpf_create_conf(ngx_cycle_t *cycle) bcf->enabled = NGX_CONF_UNSET; bcf->map_size = NGX_CONF_UNSET_UINT; + /* + * preserve environment variable value before it may be reset by + * some module, i.e. perl + */ + env = (u_char *) getenv(NGX_QUIC_BPF_VARNAME); + if (env != NULL) { + len = ngx_strlen(env); + bcf->env = ngx_pnalloc(cycle->pool, len + 1); + if (bcf->env == NULL) { + return NULL; + } + + ngx_memcpy(bcf->env, env, len + 1); + } + ngx_queue_init(&bcf->groups); return bcf; } +static char * +ngx_quic_bpf_init_conf(ngx_cycle_t *cycle, void *conf) +{ + ngx_quic_bpf_conf_t *bcf = conf; + ngx_quic_bpf_conf_t *obcf; + + ngx_conf_init_value(bcf->enabled, 0); + + if (cycle->old_cycle->conf_ctx == NULL) { + return NGX_CONF_OK; + } + + obcf = ngx_quic_bpf_get_conf(cycle->old_cycle); + if (obcf == NULL) { + return NGX_CONF_OK; + } + + if (obcf->enabled != bcf->enabled) { + ngx_log_error(NGX_LOG_WARN, cycle->log, 0, + "cannot change \"quic_bpf\" after reload, ignoring"); + bcf->enabled = obcf->enabled; + } + + bcf->master_index = obcf->master_index; + + return NGX_CONF_OK; +} + + static ngx_int_t ngx_quic_bpf_module_init(ngx_cycle_t *cycle) { - ngx_uint_t i; - ngx_listening_t *ls; - ngx_core_conf_t *ccf; - ngx_pool_cleanup_t *cln; - ngx_quic_bpf_conf_t *bcf; + ngx_uint_t i; + ngx_queue_t *q; + ngx_listening_t *ls; + ngx_core_conf_t *ccf; + ngx_pool_cleanup_t *cln; + ngx_quic_bpf_conf_t *bcf; + ngx_quic_sock_group_t *grp; if (ngx_test_config) { /* @@ -141,9 +216,21 @@ ngx_quic_bpf_module_init(ngx_cycle_t *cycle) ccf = ngx_core_get_conf(cycle); bcf = ngx_quic_bpf_get_conf(cycle); - ngx_conf_init_value(bcf->enabled, 0); + if (!bcf->enabled) { - bcf->map_size = ccf->worker_processes * 4; + if (ngx_inherited && ngx_is_init_cycle(cycle->old_cycle) + && bcf->env != NULL) + { + ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, + "\"quic_bpf\" cannot be disabled " + "during binary upgrade"); + return NGX_ERROR; + } + + return NGX_OK; + } + + bcf->map_size = ccf->worker_processes * 32; cln = ngx_pool_cleanup_add(cycle->pool, 0); if (cln == NULL) { @@ -169,6 +256,29 @@ ngx_quic_bpf_module_init(ngx_cycle_t *cycle) } } + q = ngx_queue_head(&bcf->groups); + + while (q != ngx_queue_sentinel(&bcf->groups)) { + + grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue); + q = ngx_queue_next(q); + + if (grp->listening.nelts == 0) { + + ngx_quic_bpf_close(cycle->log, grp->sock_map, "sockmap"); + ngx_quic_bpf_close(cycle->log, grp->worker_counts_map, + "worker_counts"); + ngx_queue_remove(&grp->queue); + continue; + } + + if (ngx_quic_bpf_publish_workers(cycle, grp, ccf->worker_processes) + != NGX_OK) + { + goto failed; + } + } + if (ngx_quic_bpf_export_maps(cycle) != NGX_OK) { goto failed; } @@ -203,13 +313,33 @@ failed: } +ngx_uint_t +ngx_quic_bpf_enabled(ngx_cycle_t *cycle) +{ + ngx_quic_bpf_conf_t *bcf; + + bcf = ngx_quic_bpf_get_conf(cycle); + + if (bcf == NULL) { + return 0; + } + + return bcf->enabled; +} + + static void ngx_quic_bpf_cleanup(void *data) { ngx_quic_bpf_conf_t *bcf = (ngx_quic_bpf_conf_t *) data; - ngx_queue_t *q; - ngx_quic_sock_group_t *grp; + ngx_uint_t i; + ngx_log_t *log; + ngx_queue_t *q; + ngx_quic_sock_group_t *grp; + ngx_quic_bpf_listening_t *bls; + + log = ngx_cycle->log; for (q = ngx_queue_head(&bcf->groups); q != ngx_queue_sentinel(&bcf->groups); @@ -217,7 +347,51 @@ ngx_quic_bpf_cleanup(void *data) { grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue); - ngx_quic_bpf_close(ngx_cycle->log, grp->map_fd, "map"); + bls = grp->listening.elts; + + for (i = 0; i < grp->listening.nelts; i++) { + if (bls[i].fd != (ngx_socket_t) -1) { + if (ngx_close_socket(bls[i].fd) == -1) { + ngx_log_error(NGX_LOG_ALERT, log, ngx_socket_errno, + ngx_close_socket_n " failed"); + } + } + } + + ngx_quic_bpf_close(log, grp->sock_map, "sockmap"); + ngx_quic_bpf_close(log, grp->worker_counts_map, "worker_counts"); + } +} + + +static void +ngx_quic_bpf_exit_master(ngx_cycle_t *cycle) +{ + uint32_t key, value; + ngx_queue_t *q; + ngx_quic_bpf_conf_t *bcf; + ngx_quic_sock_group_t *grp; + + bcf = ngx_quic_bpf_get_conf(cycle); + if (!bcf->enabled) { + return; + } + + key = bcf->master_index; + value = 0; + + for (q = ngx_queue_head(&bcf->groups); + q != ngx_queue_sentinel(&bcf->groups); + q = ngx_queue_next(q)) + { + grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue); + + if (ngx_quic_bpf_map_update(cycle->log, grp->worker_counts_map, + &key, &value, "worker_counts") + != NGX_OK) + { + continue; + } } } @@ -225,6 +399,10 @@ ngx_quic_bpf_cleanup(void *data) static ngx_inline void ngx_quic_bpf_close(ngx_log_t *log, int fd, const char *name) { + if (fd == -1) { + return; + } + if (close(fd) != -1) { return; } @@ -234,6 +412,21 @@ ngx_quic_bpf_close(ngx_log_t *log, int fd, const char *name) } +static ngx_inline ngx_int_t +ngx_quic_bpf_map_update(ngx_log_t *log, int fd, const void *key, + const void *value, const char *name) +{ + if (ngx_bpf_map_update(fd, key, value, BPF_ANY) != -1) { + return NGX_OK; + } + + ngx_log_error(NGX_LOG_ERR, log, ngx_errno, + "quic bpf failed to update %s map fd:%d", name, fd); + + return NGX_ERROR; +} + + static ngx_quic_sock_group_t * ngx_quic_bpf_find_group(ngx_quic_bpf_conf_t *bcf, ngx_listening_t *ls) { @@ -259,8 +452,7 @@ ngx_quic_bpf_find_group(ngx_quic_bpf_conf_t *bcf, ngx_listening_t *ls) static ngx_quic_sock_group_t * -ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle, struct sockaddr *sa, - socklen_t socklen) +ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle, ngx_listening_t *ls) { ngx_quic_bpf_conf_t *bcf; ngx_quic_sock_group_t *grp; @@ -272,12 +464,18 @@ ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle, struct sockaddr *sa, return NULL; } - grp->socklen = socklen; - grp->sockaddr = ngx_palloc(cycle->pool, socklen); - if (grp->sockaddr == NULL) { + grp->sock_map = -1; + grp->worker_counts_map = -1; + + grp->sockaddr = ls->sockaddr; + grp->socklen = ls->socklen; + + if (ngx_array_init(&grp->listening, cycle->pool, 1, + sizeof(ngx_quic_bpf_listening_t)) + != NGX_OK) + { return NULL; } - ngx_memcpy(grp->sockaddr, sa, socklen); ngx_queue_insert_tail(&bcf->groups, &grp->queue); @@ -288,47 +486,51 @@ ngx_quic_bpf_alloc_group(ngx_cycle_t *cycle, struct sockaddr *sa, static ngx_quic_sock_group_t * ngx_quic_bpf_create_group(ngx_cycle_t *cycle, ngx_listening_t *ls) { - int progfd, failed, flags, rc; + int progfd, failed; ngx_quic_bpf_conf_t *bcf; ngx_quic_sock_group_t *grp; bcf = ngx_quic_bpf_get_conf(cycle); - if (!bcf->enabled) { - return NULL; - } - - grp = ngx_quic_bpf_alloc_group(cycle, ls->sockaddr, ls->socklen); + grp = ngx_quic_bpf_alloc_group(cycle, ls); if (grp == NULL) { return NULL; } - grp->map_fd = ngx_bpf_map_create(cycle->log, BPF_MAP_TYPE_SOCKHASH, - sizeof(uint64_t), sizeof(uint64_t), - bcf->map_size, 0); - if (grp->map_fd == -1) { + grp->sock_map = ngx_bpf_map_create(cycle->log, + BPF_MAP_TYPE_SOCKHASH, + sizeof(uint64_t), + sizeof(uint64_t), + bcf->map_size, 0); + if (grp->sock_map == -1) { goto failed; } - flags = fcntl(grp->map_fd, F_GETFD); - if (flags == -1) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, errno, - "quic bpf getfd failed"); - goto failed; - } - - /* need to inherit map during binary upgrade after exec */ - flags &= ~FD_CLOEXEC; - - rc = fcntl(grp->map_fd, F_SETFD, flags); - if (rc == -1) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, errno, - "quic bpf setfd failed"); + if (ngx_quic_bpf_inherit_fd(cycle, grp->sock_map) != NGX_OK) { goto failed; } ngx_bpf_program_link(&ngx_quic_reuseport_helper, - "ngx_quic_sockmap", grp->map_fd); + "ngx_quic_sockmap", grp->sock_map); + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + "quic bpf sockmap created fd:%d", + grp->sock_map); + + grp->worker_counts_map = ngx_bpf_map_create(cycle->log, + BPF_MAP_TYPE_ARRAY, + sizeof(uint32_t), + sizeof(uint32_t), 2, 0); + if (grp->worker_counts_map == -1) { + goto failed; + } + + if (ngx_quic_bpf_inherit_fd(cycle, grp->worker_counts_map) != NGX_OK) { + goto failed; + } + + ngx_bpf_program_link(&ngx_quic_reuseport_helper, + "ngx_quic_worker_counts", grp->worker_counts_map); progfd = ngx_bpf_load_program(cycle->log, &ngx_quic_reuseport_helper); if (progfd < 0) { @@ -352,15 +554,12 @@ ngx_quic_bpf_create_group(ngx_cycle_t *cycle, ngx_listening_t *ls) goto failed; } - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, - "quic bpf sockmap created fd:%d", grp->map_fd); return grp; failed: - if (grp->map_fd != -1) { - ngx_quic_bpf_close(cycle->log, grp->map_fd, "map"); - } + ngx_quic_bpf_close(cycle->log, grp->sock_map, "sockmap"); + ngx_quic_bpf_close(cycle->log, grp->worker_counts_map, "worker_counts"); ngx_queue_remove(&grp->queue); @@ -368,6 +567,30 @@ failed: } +static ngx_int_t +ngx_quic_bpf_inherit_fd(ngx_cycle_t *cycle, int fd) +{ + int flags; + + flags = fcntl(fd, F_GETFD); + if (flags == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "quic bpf fcntl(F_GETFD) failed"); + return NGX_ERROR; + } + + flags &= ~FD_CLOEXEC; + + if (fcntl(fd, F_SETFD, flags) == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "quic bpf fcntl(F_SETFD) failed"); + return NGX_ERROR; + } + + return NGX_OK; +} + + static ngx_quic_sock_group_t * ngx_quic_bpf_get_group(ngx_cycle_t *cycle, ngx_listening_t *ls) { @@ -392,89 +615,346 @@ ngx_quic_bpf_get_group(ngx_cycle_t *cycle, ngx_listening_t *ls) return ngx_quic_bpf_create_group(cycle, ls); } - grp = ngx_quic_bpf_alloc_group(cycle, ls->sockaddr, ls->socklen); + grp = ngx_quic_bpf_alloc_group(cycle, ls); if (grp == NULL) { return NULL; } - grp->map_fd = dup(ogrp->map_fd); - if (grp->map_fd == -1) { + grp->sock_map = dup(ogrp->sock_map); + if (grp->sock_map == -1) { ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, - "quic bpf failed to duplicate bpf map descriptor"); - - ngx_queue_remove(&grp->queue); - - return NULL; + "quic bpf failed to duplicate sockmap"); + goto failed; } - ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0, - "quic bpf sockmap fd duplicated old:%d new:%d", - ogrp->map_fd, grp->map_fd); + grp->worker_counts_map = dup(ogrp->worker_counts_map); + if (grp->worker_counts_map == -1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, + "quic bpf failed to duplicate worker counts map"); + goto failed; + } return grp; + +failed: + + ngx_quic_bpf_close(cycle->log, grp->sock_map, "sockmap"); + ngx_quic_bpf_close(cycle->log, grp->worker_counts_map, "worker_counts"); + + ngx_queue_remove(&grp->queue); + + return NULL; } static ngx_int_t -ngx_quic_bpf_group_add_socket(ngx_cycle_t *cycle, ngx_listening_t *ls) +ngx_quic_bpf_group_add_socket(ngx_cycle_t *cycle, ngx_listening_t *ls) { - uint64_t cookie; - ngx_quic_bpf_conf_t *bcf; + uint64_t key, value; ngx_quic_sock_group_t *grp; - bcf = ngx_quic_bpf_get_conf(cycle); - grp = ngx_quic_bpf_get_group(cycle, ls); - if (grp == NULL) { - if (!bcf->enabled) { - return NGX_OK; - } - return NGX_ERROR; } - grp->unused = 0; - - cookie = ngx_quic_bpf_socket_key(ls->fd, cycle->log); - if (cookie == (uint64_t) NGX_ERROR) { + if (ngx_quic_bpf_add_worker_socket(cycle, grp, ls) != NGX_OK) { return NGX_ERROR; } - /* map[cookie] = socket; for use in kernel helper */ - if (ngx_bpf_map_update(grp->map_fd, &cookie, &ls->fd, BPF_ANY) == -1) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, ngx_errno, - "quic bpf failed to update socket map key=%xL", cookie); + key = ngx_quic_bpf_listen_key(ls->worker); + value = ls->fd; + + if (ngx_quic_bpf_map_update(cycle->log, grp->sock_map, + &key, &value, "listen") + != NGX_OK) + { return NGX_ERROR; } - ngx_log_debug4(NGX_LOG_DEBUG_EVENT, cycle->log, 0, - "quic bpf sockmap fd:%d add socket:%d cookie:0x%xL worker:%ui", - grp->map_fd, ls->fd, cookie, ls->worker); - - /* do not inherit this socket */ - ls->ignore = 1; - return NGX_OK; } -static uint64_t -ngx_quic_bpf_socket_key(ngx_fd_t fd, ngx_log_t *log) +static ngx_int_t +ngx_quic_bpf_publish_workers(ngx_cycle_t *cycle, ngx_quic_sock_group_t *grp, + ngx_uint_t worker_count) { - uint64_t cookie; - socklen_t optlen; + uint32_t key; + uint32_t value; + ngx_quic_bpf_conf_t *bcf; - optlen = sizeof(cookie); + bcf = ngx_quic_bpf_get_conf(cycle); - if (getsockopt(fd, SOL_SOCKET, SO_COOKIE, &cookie, &optlen) == -1) { - ngx_log_error(NGX_LOG_EMERG, log, ngx_socket_errno, - "quic bpf getsockopt(SO_COOKIE) failed"); + key = bcf->master_index; + value = worker_count; - return (ngx_uint_t) NGX_ERROR; + if (ngx_quic_bpf_map_update(cycle->log, grp->worker_counts_map, + &key, &value, "worker_counts") + != NGX_OK) + { + return NGX_ERROR; } - return cookie; + return NGX_OK; +} + + +static ngx_int_t +ngx_quic_bpf_add_worker_socket(ngx_cycle_t *cycle, ngx_quic_sock_group_t *grp, + ngx_listening_t *ls) +{ + int value; + uint64_t key, map_value; + ngx_uint_t i, n, map_updated; + ngx_socket_t s; + ngx_quic_bpf_listening_t *bls; + + s = ngx_socket(ls->sockaddr->sa_family, SOCK_DGRAM, 0); + if (s == (ngx_socket_t) -1) { + ngx_log_error(NGX_LOG_ERR, cycle->log, ngx_socket_errno, + "quic bpf " ngx_socket_n " failed"); + return NGX_ERROR; + } + + map_updated = 0; + + if (ngx_nonblocking(s) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf " ngx_nonblocking_n " failed"); + goto failed; + } + + if (fcntl(s, F_SETFD, FD_CLOEXEC) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "quic bpf fcntl(FD_CLOEXEC) failed"); + goto failed; + } + + value = 1; + + if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(SO_REUSEADDR) failed"); + goto failed; + } + + if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(SO_REUSEPORT) failed"); + goto failed; + } + + if (ls->rcvbuf != -1) { + if (setsockopt(s, SOL_SOCKET, SO_RCVBUF, + (const void *) &ls->rcvbuf, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(SO_RCVBUF, %d) failed", + ls->rcvbuf); + goto failed; + } + } + + if (ls->sndbuf != -1) { + if (setsockopt(s, SOL_SOCKET, SO_SNDBUF, + (const void *) &ls->sndbuf, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(SO_SNDBUF, %d) failed", + ls->sndbuf); + goto failed; + } + } + +#if (NGX_HAVE_IP_PKTINFO) + if (ls->wildcard && ls->sockaddr->sa_family == AF_INET) { + if (setsockopt(s, IPPROTO_IP, IP_PKTINFO, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(IP_PKTINFO) failed"); + goto failed; + } + } +#endif + +#if (NGX_HAVE_INET6 && NGX_HAVE_IPV6_RECVPKTINFO) + if (ls->wildcard && ls->sockaddr->sa_family == AF_INET6) { + if (setsockopt(s, IPPROTO_IPV6, IPV6_RECVPKTINFO, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(IPV6_RECVPKTINFO) failed"); + goto failed; + } + } +#endif + +#if (NGX_HAVE_INET6 && defined IPV6_V6ONLY) + + if (ls->sockaddr->sa_family == AF_INET6) { + int ipv6only; + + ipv6only = ls->ipv6only; + + if (setsockopt(s, IPPROTO_IPV6, IPV6_V6ONLY, + (const void *) &ipv6only, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(IPV6_V6ONLY) %V failed", + &ls->addr_text); + goto failed; + } + } + +#endif + +#if (NGX_HAVE_IP_MTU_DISCOVER) + + if (ls->sockaddr->sa_family == AF_INET) { + value = IP_PMTUDISC_DO; + + if (setsockopt(s, IPPROTO_IP, IP_MTU_DISCOVER, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(IP_MTU_DISCOVER) " + "failed, ignored"); + } + } + +#elif (NGX_HAVE_IP_DONTFRAG) + + if (ls->sockaddr->sa_family == AF_INET) { + value = 1; + + if (setsockopt(s, IPPROTO_IP, IP_DONTFRAG, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(IP_DONTFRAG) " + "failed, ignored"); + } + } + +#endif + +#if (NGX_HAVE_INET6) + +#if (NGX_HAVE_IPV6_MTU_DISCOVER) + + if (ls->sockaddr->sa_family == AF_INET6) { + value = IPV6_PMTUDISC_DO; + + if (setsockopt(s, IPPROTO_IPV6, IPV6_MTU_DISCOVER, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(IPV6_MTU_DISCOVER) " + "failed, ignored"); + } + } + +#elif (NGX_HAVE_IPV6_DONTFRAG) + + if (ls->sockaddr->sa_family == AF_INET6) { + value = 1; + + if (setsockopt(s, IPPROTO_IPV6, IPV6_DONTFRAG, + (const void *) &value, sizeof(int)) + == -1) + { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf setsockopt(IPV6_DONTFRAG) " + "failed, ignored"); + } + } + +#endif + +#endif + + if (bind(s, ls->sockaddr, ls->socklen) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf bind() failed"); + goto failed; + } + + if (RAND_bytes((u_char *) &key, sizeof(uint64_t)) != 1) { + ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, + "quic bpf RAND_bytes() failed"); + goto failed; + } + + if ((key >> 56) == 0xFF) { + key &= ~((uint64_t) 0xFF << 56); + } + + map_value = s; + if (ngx_quic_bpf_map_update(cycle->log, grp->sock_map, + &key, &map_value, "sockmap") + != NGX_OK) + { + goto failed; + } + + map_updated = 1; + + if (ls->worker >= grp->listening.nelts) { + n = ls->worker + 1 - grp->listening.nelts; + + bls = ngx_array_push_n(&grp->listening, n); + if (bls == NULL) { + goto failed; + } + + ngx_memzero(bls, n * sizeof(ngx_quic_bpf_listening_t)); + + for (i = 0; i < n; i++) { + bls[i].fd = (ngx_socket_t) -1; + } + } + + bls = grp->listening.elts; + bls[ls->worker].fd = s; + bls[ls->worker].key = key; + bls[ls->worker].listening = ls; + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + "quic bpf socket fd:%d key:%uL", s, key); + + return NGX_OK; + +failed: + + if (map_updated) { + if (ngx_bpf_map_delete(grp->sock_map, &key) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "quic bpf failed to delete map entry"); + } + } + + if (ngx_close_socket(s) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_socket_errno, + "quic bpf " ngx_close_socket_n " failed"); + } + + return NGX_ERROR; } @@ -491,31 +971,21 @@ ngx_quic_bpf_export_maps(ngx_cycle_t *cycle) ccf = ngx_core_get_conf(cycle); bcf = ngx_quic_bpf_get_conf(cycle); + if (!bcf->enabled) { + return NGX_OK; + } + + if (ngx_queue_empty(&bcf->groups)) { + return NGX_OK; + } len = sizeof(NGX_QUIC_BPF_VARNAME) + 1; - q = ngx_queue_head(&bcf->groups); - - while (q != ngx_queue_sentinel(&bcf->groups)) { - - grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue); - - q = ngx_queue_next(q); - - if (grp->unused) { - /* - * map was inherited, but it is not used in this configuration; - * do not pass such map further and drop the group to prevent - * interference with changes during reload - */ - - ngx_quic_bpf_close(cycle->log, grp->map_fd, "map"); - ngx_queue_remove(&grp->queue); - - continue; - } - - len += NGX_INT32_LEN + 1 + NGX_SOCKADDR_STRLEN + 1; + for (q = ngx_queue_head(&bcf->groups); + q != ngx_queue_sentinel(&bcf->groups); + q = ngx_queue_next(q)) + { + len += (NGX_INT32_LEN + 1) * 2 + NGX_SOCKADDR_STRLEN + 1; } len++; @@ -528,14 +998,18 @@ ngx_quic_bpf_export_maps(ngx_cycle_t *cycle) p = ngx_cpymem(buf, NGX_QUIC_BPF_VARNAME "=", sizeof(NGX_QUIC_BPF_VARNAME)); + *p++ = bcf->master_index ? '-' : '+'; + for (q = ngx_queue_head(&bcf->groups); q != ngx_queue_sentinel(&bcf->groups); q = ngx_queue_next(q)) { grp = ngx_queue_data(q, ngx_quic_sock_group_t, queue); - p = ngx_sprintf(p, "%ud", grp->map_fd); + p = ngx_sprintf(p, "%ud", grp->sock_map); + *p++ = NGX_QUIC_BPF_ADDRSEP; + p = ngx_sprintf(p, "%ud", grp->worker_counts_map); *p++ = NGX_QUIC_BPF_ADDRSEP; p += ngx_sock_ntop(grp->sockaddr, grp->socklen, p, @@ -561,90 +1035,101 @@ ngx_quic_bpf_export_maps(ngx_cycle_t *cycle) static ngx_int_t ngx_quic_bpf_import_maps(ngx_cycle_t *cycle) { - int s; + int fds[2]; u_char *inherited, *p, *v; - ngx_uint_t in_fd; + ngx_int_t fd; + ngx_uint_t i, nfd; ngx_addr_t tmp; ngx_quic_bpf_conf_t *bcf; ngx_quic_sock_group_t *grp; - inherited = (u_char *) getenv(NGX_QUIC_BPF_VARNAME); + bcf = ngx_quic_bpf_get_conf(cycle); + inherited = bcf->env; if (inherited == NULL) { return NGX_OK; } - bcf = ngx_quic_bpf_get_conf(cycle); + nfd = 0; -#if (NGX_SUPPRESS_WARN) - s = -1; -#endif + ngx_log_error(NGX_LOG_NOTICE, cycle->log, 0, + "quic bpf using inherited QUIC BPF maps from \"%s\"", + inherited); - in_fd = 1; + switch (*inherited) { - for (p = inherited, v = p; *p; p++) { + case '+': + bcf->master_index = 1; + break; + + case '-': + bcf->master_index = 0; + break; + + default: + goto failed; + } + + for (p = inherited + 1, v = p; *p; p++) { switch (*p) { case NGX_QUIC_BPF_ADDRSEP: - if (!in_fd) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, - "quic bpf failed to parse inherited env"); - return NGX_ERROR; - } - in_fd = 0; - - s = ngx_atoi(v, p - v); - if (s == NGX_ERROR) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, - "quic bpf failed to parse inherited map fd"); - return NGX_ERROR; + if (nfd > 1) { + goto failed; } + fd = ngx_atoi(v, p - v); + if (fd == NGX_ERROR) { + goto failed; + } + + fds[nfd++] = fd; v = p + 1; break; case NGX_QUIC_BPF_VARSEP: - if (in_fd) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, - "quic bpf failed to parse inherited env"); - return NGX_ERROR; + if (nfd != 2) { + goto failed; } - in_fd = 1; - grp = ngx_pcalloc(cycle->pool, - sizeof(ngx_quic_sock_group_t)); + grp = ngx_pcalloc(cycle->pool, sizeof(ngx_quic_sock_group_t)); if (grp == NULL) { - return NGX_ERROR; + goto failed; } - grp->map_fd = s; - - if (ngx_parse_addr_port(cycle->pool, &tmp, v, p - v) + if (ngx_array_init(&grp->listening, cycle->pool, 1, + sizeof(ngx_quic_bpf_listening_t)) != NGX_OK) { - ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, - "quic bpf failed to parse inherited" - " address '%*s'", p - v , v); - - ngx_quic_bpf_close(cycle->log, s, "inherited map"); - - return NGX_ERROR; + goto failed; } - grp->sockaddr = tmp.sockaddr; - grp->socklen = tmp.socklen; + grp->sock_map = fds[0]; + grp->worker_counts_map = fds[1]; - grp->unused = 1; + if (ngx_parse_addr_port(cycle->pool, &tmp, v, p - v) != NGX_OK) { + goto failed; + } + + grp->sockaddr = ngx_pcalloc(cycle->pool, tmp.socklen); + if (grp->sockaddr == NULL) { + goto failed; + } + + ngx_memcpy(grp->sockaddr, tmp.sockaddr, tmp.socklen); + grp->socklen = tmp.socklen; ngx_queue_insert_tail(&bcf->groups, &grp->queue); - ngx_log_debug3(NGX_LOG_DEBUG_EVENT, cycle->log, 0, + ngx_log_debug4(NGX_LOG_DEBUG_EVENT, cycle->log, 0, "quic bpf sockmap inherited with " - "fd:%d address:%*s", - grp->map_fd, p - v, v); + "fds:%d/%d address:%*s", + fds[0], fds[1], p - v, v); + + nfd = 0; v = p + 1; break; @@ -653,5 +1138,120 @@ ngx_quic_bpf_import_maps(ngx_cycle_t *cycle) } } + return NGX_OK; + +failed: + + for (i = 0; i < nfd; i++) { + ngx_quic_bpf_close(cycle->log, fds[i], "inherited"); + } + + ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, + "quic bpf failed to parse inherited QUIC BPF variable"); + + return NGX_ERROR; +} + + +ngx_int_t +ngx_quic_bpf_get_worker_fd(ngx_connection_t *lc, ngx_socket_t *fd) +{ + ngx_event_t *rev; + ngx_connection_t *c; + ngx_quic_bpf_conf_t *bcf; + ngx_quic_sock_group_t *grp; + ngx_quic_bpf_listening_t *bpf_listening, *bls; + + bcf = ngx_quic_bpf_get_conf((ngx_cycle_t *) ngx_cycle); + + if (!bcf->enabled || !lc->listening->quic || !lc->listening->reuseport) { + *fd = lc->fd; + return NGX_OK; + } + + grp = ngx_quic_bpf_find_group(bcf, lc->listening); + + if (grp == NULL) { + *fd = lc->fd; + return NGX_OK; + } + + if (ngx_worker >= grp->listening.nelts) { + return NGX_ERROR; + } + + bpf_listening = grp->listening.elts; + bls = &bpf_listening[ngx_worker]; + + if (bls->fd == (ngx_socket_t) -1) { + return NGX_ERROR; + } + + if (bls->connection == NULL) { + c = ngx_get_connection(bls->fd, lc->log); + if (c == NULL) { + return NGX_ERROR; + } + + c->type = SOCK_DGRAM; + c->log = lc->log; + c->listening = bls->listening; + + rev = c->read; + rev->quic = 1; + rev->log = c->log; + rev->handler = ngx_quic_recvmsg; + + if (ngx_add_event(rev, NGX_READ_EVENT, 0) == NGX_ERROR) { + ngx_free_connection(c); + return NGX_ERROR; + } + + bls->connection = c; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, lc->log, 0, + "quic bpf socket connection fd:%d", bls->fd); + } + + *fd = bls->fd; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, lc->log, 0, + "quic bpf socket fd:%d", bls->fd); + + return NGX_OK; +} + + +ngx_int_t +ngx_quic_bpf_get_worker_key(ngx_connection_t *c, uint64_t *key) +{ + ngx_quic_bpf_conf_t *bcf; + ngx_quic_sock_group_t *grp; + ngx_quic_bpf_listening_t *bls; + + bcf = ngx_quic_bpf_get_conf((ngx_cycle_t *) ngx_cycle); + + if (!bcf->enabled || !c->listening->quic || !c->listening->reuseport) { + return NGX_DECLINED; + } + + grp = ngx_quic_bpf_find_group(bcf, c->listening); + + if (grp == NULL) { + return NGX_DECLINED; + } + + if (ngx_worker >= grp->listening.nelts) { + return NGX_ERROR; + } + + bls = grp->listening.elts; + + if (bls[ngx_worker].fd == (ngx_socket_t) -1) { + return NGX_ERROR; + } + + *key = bls[ngx_worker].key; + return NGX_OK; } diff --git a/src/event/quic/ngx_event_quic_bpf.h b/src/event/quic/ngx_event_quic_bpf.h new file mode 100644 index 000000000..eaac8fb4f --- /dev/null +++ b/src/event/quic/ngx_event_quic_bpf.h @@ -0,0 +1,20 @@ + +/* + * Copyright (C) Nginx, Inc. + */ + + +#ifndef _NGX_EVENT_QUIC_BPF_H_INCLUDED_ +#define _NGX_EVENT_QUIC_BPF_H_INCLUDED_ + + +#include +#include + + +ngx_int_t ngx_quic_bpf_get_worker_fd(ngx_connection_t *lc, + ngx_socket_t *fd); +ngx_int_t ngx_quic_bpf_get_worker_key(ngx_connection_t *c, uint64_t *key); + + +#endif /* _NGX_EVENT_QUIC_BPF_H_INCLUDED_ */ diff --git a/src/event/quic/ngx_event_quic_bpf_code.c b/src/event/quic/ngx_event_quic_bpf_code.c index e32f1c68e..2461cac43 100644 --- a/src/event/quic/ngx_event_quic_bpf_code.c +++ b/src/event/quic/ngx_event_quic_bpf_code.c @@ -7,93 +7,194 @@ static ngx_bpf_reloc_t bpf_reloc_prog_ngx_quic_reuseport_helper[] = { - { "ngx_quic_sockmap", 77 }, + { "ngx_quic_sockmap", 81 }, + { "ngx_quic_worker_counts", 97 }, + { "ngx_quic_worker_counts", 106 }, + { "ngx_quic_sockmap", 133 }, + { "ngx_quic_sockmap", 170 }, }; static struct bpf_insn bpf_insn_prog_ngx_quic_reuseport_helper[] = { /* opcode dst src offset imm */ { 0xbf, BPF_REG_6, BPF_REG_1, (int16_t) 0, 0x0 }, - { 0x79, BPF_REG_8, BPF_REG_6, (int16_t) 8, 0x0 }, - { 0x79, BPF_REG_7, BPF_REG_6, (int16_t) 0, 0x0 }, - { 0xbf, BPF_REG_1, BPF_REG_7, (int16_t) 0, 0x0 }, + { 0x79, BPF_REG_9, BPF_REG_6, (int16_t) 8, 0x0 }, + { 0x79, BPF_REG_8, BPF_REG_6, (int16_t) 0, 0x0 }, + { 0xbf, BPF_REG_1, BPF_REG_8, (int16_t) 0, 0x0 }, { 0x7, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x8 }, - { 0x2d, BPF_REG_8, BPF_REG_1, (int16_t) 10, 0x0 }, + { 0x2d, BPF_REG_9, BPF_REG_1, (int16_t) 11, 0x0 }, { 0xbf, BPF_REG_3, BPF_REG_10, (int16_t) 0, 0x0 }, - { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xfffffff7 }, + { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xffffffef }, { 0xbf, BPF_REG_1, BPF_REG_6, (int16_t) 0, 0x0 }, { 0xb7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x8 }, { 0xb7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x1 }, { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x1a }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 }, { 0x15, BPF_REG_0, BPF_REG_0, (int16_t) 1, 0x0 }, - { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 67, 0x0 }, - { 0x71, BPF_REG_1, BPF_REG_10, (int16_t) 65527, 0x0 }, + { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 163, 0x0 }, + { 0x71, BPF_REG_1, BPF_REG_10, (int16_t) 65519, 0x0 }, { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 2, 0x0 }, { 0x71, BPF_REG_1, BPF_REG_1, (int16_t) 0, 0x0 }, - { 0x73, BPF_REG_10, BPF_REG_1, (int16_t) 65527, 0x0 }, + { 0x73, BPF_REG_10, BPF_REG_1, (int16_t) 65519, 0x0 }, { 0xb7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x9 }, { 0x67, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x38 }, { 0xc7, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x38 }, { 0x65, BPF_REG_1, BPF_REG_0, (int16_t) 18, 0xffffffff }, - { 0xbf, BPF_REG_1, BPF_REG_7, (int16_t) 0, 0x0 }, + { 0xbf, BPF_REG_1, BPF_REG_8, (int16_t) 0, 0x0 }, { 0x7, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0xd }, - { 0x2d, BPF_REG_8, BPF_REG_1, (int16_t) 11, 0x0 }, + { 0x2d, BPF_REG_9, BPF_REG_1, (int16_t) 11, 0x0 }, { 0xbf, BPF_REG_3, BPF_REG_10, (int16_t) 0, 0x0 }, - { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xfffffff7 }, + { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xffffffef }, { 0xbf, BPF_REG_1, BPF_REG_6, (int16_t) 0, 0x0 }, { 0xb7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xd }, { 0xb7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x1 }, { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x1a }, - { 0x55, BPF_REG_0, BPF_REG_0, (int16_t) 49, 0x0 }, - { 0xb7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xe }, - { 0x71, BPF_REG_1, BPF_REG_10, (int16_t) 65527, 0x0 }, - { 0x15, BPF_REG_1, BPF_REG_0, (int16_t) 5, 0x14 }, - { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 45, 0x0 }, - { 0xb7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xe }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x15, BPF_REG_0, BPF_REG_0, (int16_t) 1, 0x0 }, + { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 143, 0x0 }, + { 0x71, BPF_REG_1, BPF_REG_10, (int16_t) 65519, 0x0 }, + { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 2, 0x0 }, { 0x71, BPF_REG_1, BPF_REG_1, (int16_t) 0, 0x0 }, - { 0x73, BPF_REG_10, BPF_REG_1, (int16_t) 65527, 0x0 }, - { 0x55, BPF_REG_1, BPF_REG_0, (int16_t) 41, 0x14 }, - { 0xf, BPF_REG_7, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0xbf, BPF_REG_1, BPF_REG_7, (int16_t) 0, 0x0 }, + { 0x73, BPF_REG_10, BPF_REG_1, (int16_t) 65519, 0x0 }, + { 0xb7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xe }, + { 0x55, BPF_REG_1, BPF_REG_0, (int16_t) 52, 0x14 }, + { 0xf, BPF_REG_8, BPF_REG_2, (int16_t) 0, 0x0 }, + { 0xbf, BPF_REG_1, BPF_REG_8, (int16_t) 0, 0x0 }, { 0x7, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x14 }, - { 0x3d, BPF_REG_8, BPF_REG_1, (int16_t) 7, 0x0 }, - { 0xbf, BPF_REG_7, BPF_REG_10, (int16_t) 0, 0x0 }, - { 0x7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0xffffffe3 }, + { 0x3d, BPF_REG_9, BPF_REG_1, (int16_t) 8, 0x0 }, + { 0xbf, BPF_REG_8, BPF_REG_10, (int16_t) 0, 0x0 }, + { 0x7, BPF_REG_8, BPF_REG_0, (int16_t) 0, 0xffffffdb }, { 0xbf, BPF_REG_1, BPF_REG_6, (int16_t) 0, 0x0 }, - { 0xbf, BPF_REG_3, BPF_REG_7, (int16_t) 0, 0x0 }, + { 0xbf, BPF_REG_3, BPF_REG_8, (int16_t) 0, 0x0 }, { 0xb7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x14 }, { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x1a }, - { 0x55, BPF_REG_0, BPF_REG_0, (int16_t) 30, 0x0 }, - { 0x71, BPF_REG_2, BPF_REG_7, (int16_t) 0, 0x0 }, - { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x38 }, - { 0x71, BPF_REG_1, BPF_REG_7, (int16_t) 1, 0x0 }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x55, BPF_REG_0, BPF_REG_0, (int16_t) 125, 0x0 }, + { 0x71, BPF_REG_1, BPF_REG_8, (int16_t) 1, 0x0 }, { 0x67, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x30 }, - { 0x4f, BPF_REG_1, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0x71, BPF_REG_2, BPF_REG_7, (int16_t) 2, 0x0 }, - { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x28 }, - { 0x4f, BPF_REG_1, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0x71, BPF_REG_2, BPF_REG_7, (int16_t) 3, 0x0 }, - { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x20 }, - { 0x4f, BPF_REG_1, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0x71, BPF_REG_2, BPF_REG_7, (int16_t) 4, 0x0 }, - { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x18 }, - { 0x4f, BPF_REG_1, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0x71, BPF_REG_2, BPF_REG_7, (int16_t) 5, 0x0 }, - { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x10 }, - { 0x4f, BPF_REG_1, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0x71, BPF_REG_2, BPF_REG_7, (int16_t) 6, 0x0 }, - { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x8 }, - { 0x4f, BPF_REG_1, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0x71, BPF_REG_2, BPF_REG_7, (int16_t) 7, 0x0 }, - { 0x4f, BPF_REG_1, BPF_REG_2, (int16_t) 0, 0x0 }, - { 0x7b, BPF_REG_10, BPF_REG_1, (int16_t) 65528, 0x0 }, + { 0x71, BPF_REG_2, BPF_REG_8, (int16_t) 0, 0x0 }, + { 0xbf, BPF_REG_3, BPF_REG_2, (int16_t) 0, 0x0 }, + { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x38 }, + { 0x4f, BPF_REG_1, BPF_REG_3, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_3, BPF_REG_8, (int16_t) 2, 0x0 }, + { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x28 }, + { 0x4f, BPF_REG_1, BPF_REG_3, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_3, BPF_REG_8, (int16_t) 3, 0x0 }, + { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x4f, BPF_REG_1, BPF_REG_3, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_3, BPF_REG_8, (int16_t) 4, 0x0 }, + { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x18 }, + { 0x4f, BPF_REG_1, BPF_REG_3, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_3, BPF_REG_8, (int16_t) 5, 0x0 }, + { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x10 }, + { 0x4f, BPF_REG_1, BPF_REG_3, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_3, BPF_REG_8, (int16_t) 7, 0x0 }, + { 0x4f, BPF_REG_1, BPF_REG_3, (int16_t) 0, 0x0 }, + { 0x71, BPF_REG_3, BPF_REG_8, (int16_t) 6, 0x0 }, + { 0x67, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0x8 }, + { 0x4f, BPF_REG_1, BPF_REG_3, (int16_t) 0, 0x0 }, + { 0x7b, BPF_REG_10, BPF_REG_1, (int16_t) 65520, 0x0 }, + { 0x15, BPF_REG_2, BPF_REG_0, (int16_t) 15, 0xff }, { 0xbf, BPF_REG_3, BPF_REG_10, (int16_t) 0, 0x0 }, - { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xfffffff8 }, + { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xfffffff0 }, { 0xbf, BPF_REG_1, BPF_REG_6, (int16_t) 0, 0x0 }, { 0x18, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x0 }, { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, { 0xb7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x0 }, { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x52 }, - { 0xb7, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x1 }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x1 }, + { 0x67, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x77, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x15, BPF_REG_0, BPF_REG_0, (int16_t) 89, 0x0 }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0xfffffffe }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x1d, BPF_REG_0, BPF_REG_1, (int16_t) 1, 0x0 }, + { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 84, 0x0 }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x63, BPF_REG_10, BPF_REG_7, (int16_t) 65532, 0x0 }, + { 0xbf, BPF_REG_2, BPF_REG_10, (int16_t) 0, 0x0 }, + { 0x7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xfffffffc }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x1 }, + { 0xbf, BPF_REG_8, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x15, BPF_REG_8, BPF_REG_0, (int16_t) 76, 0x0 }, + { 0xb7, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x1 }, + { 0x63, BPF_REG_10, BPF_REG_1, (int16_t) 65532, 0x0 }, + { 0xbf, BPF_REG_2, BPF_REG_10, (int16_t) 0, 0x0 }, + { 0x7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0xfffffffc }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x1 }, + { 0x15, BPF_REG_0, BPF_REG_0, (int16_t) 68, 0x0 }, + { 0x61, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x61, BPF_REG_9, BPF_REG_8, (int16_t) 0, 0x0 }, + { 0x2d, BPF_REG_9, BPF_REG_1, (int16_t) 1, 0x0 }, + { 0xbf, BPF_REG_9, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x15, BPF_REG_9, BPF_REG_0, (int16_t) 63, 0x0 }, + { 0xbf, BPF_REG_1, BPF_REG_9, (int16_t) 0, 0x0 }, + { 0x67, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x77, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0xbf, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x61, BPF_REG_2, BPF_REG_6, (int16_t) 32, 0x0 }, + { 0x9f, BPF_REG_2, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0xc7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0xffffffff }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0xffff }, + { 0x5f, BPF_REG_2, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0xff000000 }, + { 0x4f, BPF_REG_2, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x7b, BPF_REG_10, BPF_REG_2, (int16_t) 65520, 0x0 }, + { 0xbf, BPF_REG_3, BPF_REG_10, (int16_t) 0, 0x0 }, + { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xfffffff0 }, + { 0xbf, BPF_REG_1, BPF_REG_6, (int16_t) 0, 0x0 }, + { 0x18, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0xb7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x52 }, + { 0xbf, BPF_REG_2, BPF_REG_7, (int16_t) 0, 0x0 }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x1 }, + { 0x67, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x77, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x15, BPF_REG_0, BPF_REG_0, (int16_t) 36, 0x0 }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0xfffffffe }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x1d, BPF_REG_0, BPF_REG_1, (int16_t) 1, 0x0 }, + { 0x5, BPF_REG_0, BPF_REG_0, (int16_t) 31, 0x0 }, + { 0x61, BPF_REG_1, BPF_REG_8, (int16_t) 0, 0x0 }, + { 0x61, BPF_REG_2, BPF_REG_2, (int16_t) 0, 0x0 }, + { 0x2d, BPF_REG_2, BPF_REG_1, (int16_t) 1, 0x0 }, + { 0xbf, BPF_REG_1, BPF_REG_2, (int16_t) 0, 0x0 }, + { 0x15, BPF_REG_1, BPF_REG_0, (int16_t) 26, 0x0 }, + { 0x67, BPF_REG_9, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x77, BPF_REG_9, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x67, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x77, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x1d, BPF_REG_1, BPF_REG_9, (int16_t) 21, 0x0 }, + { 0x61, BPF_REG_2, BPF_REG_6, (int16_t) 32, 0x0 }, + { 0x9f, BPF_REG_2, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x67, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0xc7, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0xffffffff }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0xffff }, + { 0x5f, BPF_REG_2, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x18, BPF_REG_1, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0xff000000 }, + { 0x4f, BPF_REG_2, BPF_REG_1, (int16_t) 0, 0x0 }, + { 0x7b, BPF_REG_10, BPF_REG_2, (int16_t) 65520, 0x0 }, + { 0xbf, BPF_REG_3, BPF_REG_10, (int16_t) 0, 0x0 }, + { 0x7, BPF_REG_3, BPF_REG_0, (int16_t) 0, 0xfffffff0 }, + { 0xbf, BPF_REG_1, BPF_REG_6, (int16_t) 0, 0x0 }, + { 0x18, BPF_REG_2, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x0, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0xb7, BPF_REG_4, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0x85, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x52 }, + { 0x67, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x77, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x20 }, + { 0x15, BPF_REG_0, BPF_REG_0, (int16_t) 1, 0x0 }, + { 0xb7, BPF_REG_7, BPF_REG_0, (int16_t) 0, 0x0 }, + { 0xbf, BPF_REG_0, BPF_REG_7, (int16_t) 0, 0x0 }, { 0x95, BPF_REG_0, BPF_REG_0, (int16_t) 0, 0x0 }, }; diff --git a/src/event/quic/ngx_event_quic_connection.h b/src/event/quic/ngx_event_quic_connection.h index efcb632ca..40873cf1e 100644 --- a/src/event/quic/ngx_event_quic_connection.h +++ b/src/event/quic/ngx_event_quic_connection.h @@ -48,6 +48,9 @@ typedef struct ngx_quic_keys_s ngx_quic_keys_t; #include #include #include +#if (NGX_QUIC_BPF) +#include +#endif /* RFC 9002, 6.2.2. Handshakes and New Paths: kInitialRtt */ diff --git a/src/event/quic/ngx_event_quic_connid.c b/src/event/quic/ngx_event_quic_connid.c index 4e7b8dc22..b76eaaa5f 100644 --- a/src/event/quic/ngx_event_quic_connid.c +++ b/src/event/quic/ngx_event_quic_connid.c @@ -34,7 +34,7 @@ ngx_quic_create_server_id(ngx_connection_t *c, u_char *id) if (ngx_quic_bpf_attach_id(c, id) != NGX_OK) { ngx_log_error(NGX_LOG_ERR, c->log, 0, "quic bpf failed to generate socket key"); - /* ignore error, things still may work */ + return NGX_ERROR; } #endif @@ -47,22 +47,21 @@ ngx_quic_create_server_id(ngx_connection_t *c, u_char *id) static ngx_int_t ngx_quic_bpf_attach_id(ngx_connection_t *c, u_char *id) { - int fd; - uint64_t cookie; - socklen_t optlen; + uint64_t key; + ngx_int_t rc; - fd = c->listening->fd; + rc = ngx_quic_bpf_get_worker_key(c, &key); - optlen = sizeof(cookie); - - if (getsockopt(fd, SOL_SOCKET, SO_COOKIE, &cookie, &optlen) == -1) { - ngx_log_error(NGX_LOG_ERR, c->log, ngx_socket_errno, - "quic getsockopt(SO_COOKIE) failed"); + if (rc == NGX_DECLINED) { + /* BPF not applicable for this listener, leave DCID as random */ + return NGX_OK; + } + if (rc != NGX_OK) { return NGX_ERROR; } - ngx_quic_dcid_encode_key(id, cookie); + ngx_quic_dcid_encode_key(id, key); return NGX_OK; } diff --git a/src/event/quic/ngx_event_quic_output.c b/src/event/quic/ngx_event_quic_output.c index f98c834a1..cb75d2868 100644 --- a/src/event/quic/ngx_event_quic_output.c +++ b/src/event/quic/ngx_event_quic_output.c @@ -1085,7 +1085,6 @@ ngx_quic_send_retry(ngx_connection_t *c, ngx_quic_conf_t *conf, pkt.odcid = inpkt->dcid; pkt.dcid = inpkt->scid; - /* TODO: generate routable dcid */ if (RAND_bytes(dcid, NGX_QUIC_SERVER_CID_LEN) != 1) { return NGX_ERROR; } diff --git a/src/event/quic/ngx_event_quic_udp.c b/src/event/quic/ngx_event_quic_udp.c index 15b54bc82..46c741ce6 100644 --- a/src/event/quic/ngx_event_quic_udp.c +++ b/src/event/quic/ngx_event_quic_udp.c @@ -26,6 +26,7 @@ ngx_quic_recvmsg(ngx_event_t *ev) ngx_err_t err; socklen_t socklen, local_socklen; ngx_event_t *rev, *wev; + ngx_socket_t cfd; struct iovec iov[1]; struct msghdr msg; ngx_sockaddr_t sa, lsa; @@ -168,7 +169,7 @@ ngx_quic_recvmsg(ngx_event_t *ev) c->log->handler = NULL; ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, - "quic recvmsg: fd:%d n:%z", c->fd, n); + "quic recvmsg: fd:%d n:%z", lc->fd, n); c->log->handler = handler; } @@ -204,6 +205,14 @@ ngx_quic_recvmsg(ngx_event_t *ev) goto next; } +#if (NGX_QUIC_BPF) + if (lc->read->quic) { + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0, + "quic new connection on worker socket, dropped"); + goto next; + } +#endif + #if (NGX_STAT_STUB) (void) ngx_atomic_fetch_add(ngx_stat_accepted, 1); #endif @@ -211,7 +220,15 @@ ngx_quic_recvmsg(ngx_event_t *ev) ngx_accept_disabled = ngx_cycle->connection_n / 8 - ngx_cycle->free_connection_n; - c = ngx_get_connection(lc->fd, ev->log); + cfd = lc->fd; + +#if (NGX_QUIC_BPF) + if (ngx_quic_bpf_get_worker_fd(lc, &cfd) != NGX_OK) { + return; + } +#endif + + c = ngx_get_connection(cfd, ev->log); if (c == NULL) { return; } @@ -327,7 +344,7 @@ ngx_quic_recvmsg(ngx_event_t *ev) ngx_log_debug4(NGX_LOG_DEBUG_EVENT, log, 0, "*%uA quic recvmsg: %V fd:%d n:%z", - c->number, &addr, c->fd, n); + c->number, &addr, lc->fd, n); } } diff --git a/src/os/unix/ngx_process_cycle.c b/src/os/unix/ngx_process_cycle.c index 22a0e07f2..ede5ae0ba 100644 --- a/src/os/unix/ngx_process_cycle.c +++ b/src/os/unix/ngx_process_cycle.c @@ -989,7 +989,8 @@ ngx_worker_process_exit(ngx_cycle_t *cycle) && c[i].read && !c[i].read->accept && !c[i].read->channel - && !c[i].read->resolver) + && !c[i].read->resolver + && !c[i].read->quic) { ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "*%uA open socket #%d left in connection %ui",