https.c

Go to the documentation of this file.

#define _POSIX_C_SOURCE 200809L
 
#include <cwist/net/http/https.h>
#include <cwist/net/http/writer_fast.h>
#include <cwist/core/sstring/sstring.h>
#include <cwist/sys/err/cwist_err.h>
#include <cwist/core/mem/alloc.h>
#include <cwist/core/mem/gc.h>
#include <cwist/sys/app/shutdown.h>
#include "tls_chain.h"
#include "async_internal.h"
#include <openssl/ssl.h>
#include <openssl/err.h>
#include <openssl/evp.h>
#include <openssl/hmac.h>
#include <openssl/rand.h>
#include <openssl/crypto.h>
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
#include <string.h>
#include <errno.h>
#include <poll.h>
#ifdef __linux__
#include <sys/epoll.h>
#endif
#include <fcntl.h>
#include <sys/socket.h>
#include <arpa/inet.h>
#include <netinet/tcp.h>
#include <stdint.h>
#include <pthread.h>
#include <stdbool.h>
#include <stdatomic.h>
#include <time.h>
 
/* Forward declaration: PQC layer applied inside TLS bootstrap */
bool cwist_tls_apply_pqc_layer(cwist_app *app, SSL_CTX *ctx);
 
/* Bodies up to this size are written in the same TLS record as the
 * response headers; larger bodies amortize the per-record overhead. */
#define CWIST_TLS_COALESCE_MAX (16 * 1024)
 
static uint64_t cwist_https_now_ms(void) {
    struct timespec ts;
    clock_gettime(CLOCK_MONOTONIC, &ts);
    return (uint64_t)ts.tv_sec * 1000 + (uint64_t)ts.tv_nsec / 1000000;
}
 
static int cwist_ssl_wait(int fd, int ssl_error, int timeout_ms) {
    struct pollfd pfd = {.fd = fd, .events = 0};
    if (ssl_error == SSL_ERROR_WANT_READ) {
        pfd.events = POLLIN;
    } else if (ssl_error == SSL_ERROR_WANT_WRITE) {
        pfd.events = POLLOUT;
    } else {
        return -1;
    }
 
    int ret = poll(&pfd, 1, timeout_ms);
    if (ret <= 0) return -1;
    if (pfd.revents & (POLLERR | POLLHUP | POLLNVAL)) return -1;
    return 0;
}
 
struct https_thread_payload {
    int client_fd;
    cwist_https_context *ctx;
    void (*handler)(cwist_https_connection *, void *);
    void *user_ctx;
    SSL *pres_ssl;  /* non-NULL when the shepherd already finished the handshake */
    cwist_https_connection *conn; /* non-NULL when re-arming an established connection */
};
 
/* --- Thread Pool for HTTPS --- */
/* The task queue is a ring that starts small and doubles when a burst fills
 * it, up to HTTPS_TASK_QUEUE_MAX tasks; past that, submitters block until a
 * worker frees a slot (the old fixed-size behaviour). It used to be a static
 * HTTPS_TASK_QUEUE_MAX-entry array (~96 MiB) that https_pool_init()'s memset
 * made resident in every worker process before the first connection. The
 * ring never shrinks: a process keeps the capacity of its largest burst. */
#define HTTPS_TASK_QUEUE_INITIAL 1024
#define HTTPS_TASK_QUEUE_MAX 2097152
 
typedef struct {
    int client_fd;
    cwist_https_context *ctx;
    void (*handler)(cwist_https_connection *, void *);
    void *user_ctx;
    SSL *pres_ssl;
    cwist_https_connection *conn;
} https_pool_task_t;
 
typedef struct {
    pthread_t threads[2048];
    size_t threads_size;
    https_pool_task_t *queue; /* ring of queue_len slots; cwist_malloc'd, never GC-tracked */
    size_t queue_len;
    size_t head;
    size_t tail;
    size_t count;
    pthread_mutex_t mutex;
    pthread_cond_t cond_not_empty;
    pthread_cond_t cond_not_full;
    int shutdown;
    bool resizing; /* a submitter is allocating a bigger ring outside the mutex */
} https_thread_pool_t;
 
static https_thread_pool_t g_https_pool;
static bool g_https_pool_initialized = false;
 
// Forward declaration of existing https_thread_handler
static void *https_thread_handler(void *arg);
void https_pool_submit_ready(int client_fd, SSL *pres_ssl, cwist_https_context *ctx,
                             void (*handler)(cwist_https_connection *, void *), void *user_ctx);
static cwist_error_t https_wrap_established(cwist_https_context *ctx, int client_fd, SSL *ssl,
                                            cwist_https_connection **conn);
int https_hs_shepherd_start(void);
void https_hs_shepherd_stop(void);
static void https_park_stop(void);
static void https_park_reopen(void);
static void https_connection_teardown(cwist_https_connection *conn);
static void https_park_teardown(cwist_https_connection *conn);
/* Set by cwist_https_park() on the pool thread that parked its connection:
 * the connection may already be served by another thread, so the pool
 * thread must not touch it (or close it) after the handler returns. */
static __thread bool t_https_parked = false;
static void https_close_conn_cb(void *handle);
 
static inline https_pool_task_t *resize_queue(https_pool_task_t *queue, size_t new_len) {
    https_pool_task_t *old = g_https_pool.queue;
    size_t count = g_https_pool.count;
    if (count > 0) {
        /* Two contiguous copies instead of a per-slot modulo walk: this is
         * the only part of a resize that still runs under the mutex. */
        size_t first = g_https_pool.queue_len - g_https_pool.head;
        if (first > count) first = count;
        memcpy(queue, old + g_https_pool.head, first * sizeof(*queue));
        memcpy(queue + first, old, (count - first) * sizeof(*queue));
    }
    g_https_pool.queue = queue;
    g_https_pool.queue_len = new_len;
    g_https_pool.head = 0;
    g_https_pool.tail = count;
    return old;
}
 
static bool https_pool_reserve_locked(void) {
    while (g_https_pool.count >= g_https_pool.queue_len) {
        if (g_https_pool.resizing || g_https_pool.shutdown) return false;
        if (g_https_pool.queue_len >= HTTPS_TASK_QUEUE_MAX) return false;
        size_t new_len = g_https_pool.queue_len ? g_https_pool.queue_len * 2 : HTTPS_TASK_QUEUE_INITIAL;
        if (new_len > HTTPS_TASK_QUEUE_MAX) new_len = HTTPS_TASK_QUEUE_MAX;
 
        g_https_pool.resizing = true;
        pthread_mutex_unlock(&g_https_pool.mutex);
        /* cwist_malloc, not cwist_alloc: every pool thread touches the ring,
         * so it must not sit on one thread's full-GC sweep list. */
        https_pool_task_t *queue = cwist_malloc(new_len * sizeof(*queue));
        pthread_mutex_lock(&g_https_pool.mutex);
        if (!queue || g_https_pool.shutdown) {
            /* On shutdown https_pool_destroy() owns the ring; drop ours. */
            cwist_free(queue);
            g_https_pool.resizing = false;
            pthread_cond_broadcast(&g_https_pool.cond_not_full);
            return false;
        }
 
        /* Only this submitter grows the ring and nothing shrinks it, so
         * queue_len is unchanged and new_len still exceeds count. */
        https_pool_task_t *old = resize_queue(queue, new_len);
        pthread_cond_broadcast(&g_https_pool.cond_not_full); /* slots are free now */
        pthread_mutex_unlock(&g_https_pool.mutex);
        cwist_free(old);
        pthread_mutex_lock(&g_https_pool.mutex);
        /* resizing stays set until here so https_pool_destroy() cannot tear
         * down the mutex while this thread is still outside it. */
        g_https_pool.resizing = false;
        pthread_cond_broadcast(&g_https_pool.cond_not_full);
        /* Others may have filled the new slots meanwhile: loop rechecks. */
    }
    return true;
}
 
static void *https_pool_worker(void *arg) {
    (void)arg;
    while (1) {
        pthread_mutex_lock(&g_https_pool.mutex);
        while (g_https_pool.count == 0 && !g_https_pool.shutdown) {
            pthread_cond_wait(&g_https_pool.cond_not_empty, &g_https_pool.mutex);
        }
        if (g_https_pool.shutdown) {
            pthread_mutex_unlock(&g_https_pool.mutex);
            break;
        }
        https_pool_task_t task = g_https_pool.queue[g_https_pool.head];
        g_https_pool.head = (g_https_pool.head + 1) % g_https_pool.queue_len;
        g_https_pool.count--;
        pthread_cond_signal(&g_https_pool.cond_not_full);
        pthread_mutex_unlock(&g_https_pool.mutex);
 
        /* A resubmitted or resumed connection was tracked by whichever
         * thread wrapped it; this thread serves it now, so its exit sweep
         * (not the tracker's) is the one that may close it (#345). */
        if (task.conn && cwist_full_gc_enabled()) cwist_conn_registry_adopt(task.conn);
 
        struct https_thread_payload *payload = cwist_alloc(sizeof(*payload));
        if (payload) {
            payload->client_fd = task.client_fd;
            payload->ctx = task.ctx;
            payload->handler = task.handler;
            payload->user_ctx = task.user_ctx;
            payload->pres_ssl = task.pres_ssl;
            payload->conn = task.conn;
            https_thread_handler(payload);
        } else {
            if (task.conn) {
                cwist_https_close_connection(task.conn);
            } else {
                if (task.pres_ssl) SSL_free(task.pres_ssl);
                close(task.client_fd);
            }
        }
    }
    return NULL;
}
 
int https_pool_init(void) {
    if (g_https_pool_initialized) return 0;
    https_park_reopen();
    memset(&g_https_pool, 0, sizeof(g_https_pool));
    pthread_mutex_init(&g_https_pool.mutex, NULL);
    pthread_cond_init(&g_https_pool.cond_not_empty, NULL);
    pthread_cond_init(&g_https_pool.cond_not_full, NULL);
    for (int i = 0; i < get_optimal_thread_count(); i++) {
        if (pthread_create(&g_https_pool.threads[i], NULL, https_pool_worker, NULL) != 0) {
            return -1;
        }
    }
    g_https_pool_initialized = true;
    return 0;
}
 
void https_pool_submit(int client_fd, cwist_https_context *ctx,
                       void (*handler)(cwist_https_connection *, void *), void *user_ctx) {
    https_pool_submit_ready(client_fd, NULL, ctx, handler, user_ctx);
}
 
void https_pool_submit_ready(int client_fd, SSL *pres_ssl, cwist_https_context *ctx,
                             void (*handler)(cwist_https_connection *, void *), void *user_ctx) {
    pthread_mutex_lock(&g_https_pool.mutex);
    while (!g_https_pool.shutdown && !https_pool_reserve_locked()) {
        pthread_cond_wait(&g_https_pool.cond_not_full, &g_https_pool.mutex);
    }
    if (g_https_pool.shutdown) {
        pthread_mutex_unlock(&g_https_pool.mutex);
        if (pres_ssl) SSL_free(pres_ssl);
        close(client_fd);
        return;
    }
    g_https_pool.queue[g_https_pool.tail].client_fd = client_fd;
    g_https_pool.queue[g_https_pool.tail].ctx = ctx;
    g_https_pool.queue[g_https_pool.tail].handler = handler;
    g_https_pool.queue[g_https_pool.tail].user_ctx = user_ctx;
    g_https_pool.queue[g_https_pool.tail].pres_ssl = pres_ssl;
    g_https_pool.queue[g_https_pool.tail].conn = NULL;
    g_https_pool.tail = (g_https_pool.tail + 1) % g_https_pool.queue_len;
    g_https_pool.count++;
    pthread_cond_signal(&g_https_pool.cond_not_empty);
    pthread_mutex_unlock(&g_https_pool.mutex);
}
 
void https_pool_submit_conn(cwist_https_connection *conn, cwist_https_context *ctx,
                            void (*handler)(cwist_https_connection *, void *), void *user_ctx) {
    if (!conn) return;
    pthread_mutex_lock(&g_https_pool.mutex);
    while (!g_https_pool.shutdown && !https_pool_reserve_locked()) {
        pthread_cond_wait(&g_https_pool.cond_not_full, &g_https_pool.mutex);
    }
    if (g_https_pool.shutdown) {
        pthread_mutex_unlock(&g_https_pool.mutex);
        cwist_https_close_connection(conn);
        return;
    }
    g_https_pool.queue[g_https_pool.tail].client_fd = conn->fd;
    g_https_pool.queue[g_https_pool.tail].ctx = ctx;
    g_https_pool.queue[g_https_pool.tail].handler = handler;
    g_https_pool.queue[g_https_pool.tail].user_ctx = user_ctx;
    g_https_pool.queue[g_https_pool.tail].pres_ssl = NULL;
    g_https_pool.queue[g_https_pool.tail].conn = conn;
    g_https_pool.tail = (g_https_pool.tail + 1) % g_https_pool.queue_len;
    g_https_pool.count++;
    pthread_cond_signal(&g_https_pool.cond_not_empty);
    pthread_mutex_unlock(&g_https_pool.mutex);
}
 
void https_pool_destroy(void) {
    if (!g_https_pool_initialized) return;
    https_hs_shepherd_stop();
    https_park_stop();
    pthread_mutex_lock(&g_https_pool.mutex);
    g_https_pool.shutdown = 1;
    pthread_cond_broadcast(&g_https_pool.cond_not_empty);
    /* Wake submitters blocked on a full ring so they see shutdown, and wait
     * out a submitter that is growing the ring outside the mutex: the ring
     * and the mutex are torn down below. */
    pthread_cond_broadcast(&g_https_pool.cond_not_full);
    while (g_https_pool.resizing) pthread_cond_wait(&g_https_pool.cond_not_full, &g_https_pool.mutex);
    pthread_mutex_unlock(&g_https_pool.mutex);
    for (int i = 0; i < get_optimal_thread_count(); i++) {
        pthread_join(g_https_pool.threads[i], NULL);
    }
    while (g_https_pool.count > 0) {
        https_pool_task_t task = g_https_pool.queue[g_https_pool.head];
        g_https_pool.head = (g_https_pool.head + 1) % g_https_pool.queue_len;
        g_https_pool.count--;
        if (task.conn) {
            cwist_https_close_connection(task.conn);
        } else {
            if (task.pres_ssl) SSL_free(task.pres_ssl);
            if (task.client_fd >= 0) close(task.client_fd);
        }
    }
    pthread_mutex_destroy(&g_https_pool.mutex);
    pthread_cond_destroy(&g_https_pool.cond_not_empty);
    pthread_cond_destroy(&g_https_pool.cond_not_full);
    cwist_free(g_https_pool.queue);
    g_https_pool.queue = NULL;
    g_https_pool.queue_len = 0;
 
    g_https_pool_initialized = false;
}
/* --- End Thread Pool --- */
 
/* --- Idle connection park set ---------------------------------------------
 * Why this exists: a pool thread serving a TLS connection used to wait in
 * poll() for that connection's next request (30 s for HTTP/1.1 keep-alive,
 * up to the HTTP/2 idle timeout).  An idle client therefore held a whole
 * pool thread, and with one or two pool threads per worker only a few dozen
 * TLS connections could be served at once; the rest queued behind idle
 * ones.  A pool thread with nothing to read now parks the connection here
 * and returns.  One epoll thread per process watches every parked fd and
 * resubmits a connection to the pool when bytes arrive, so an idle
 * connection costs memory, not a thread.
 *
 * Each class keeps a FIFO list; every entry of a class gets the same idle
 * budget at park time, so the list is (nearly) ordered by deadline and
 * expiry only looks at the head.  HTTP/1.1 entries are closed on expiry,
 * exactly as the blocking header read closed them.  HTTP/2 entries are
 * resubmitted with park_expired set so the session sends GOAWAY and keeps
 * its grace window, as before.
 *
 * Parking is disabled under full GC: its allocations are owned per thread,
 * and a parked HTTP/2 session would move between threads with them.
 * ------------------------------------------------------------------------- */
#ifdef __linux__
typedef struct https_parked {
    cwist_https_connection *conn;
    uint64_t deadline_ms;
    int cls;
    bool linked;
    struct https_parked *prev, *next;
} https_parked_t;
 
static struct {
    pthread_mutex_t lock;
    pthread_t thread;
    int epoll_fd;
    bool running;
    /* Set by https_park_stop() until the next https_pool_init(): pool
     * workers still run while the pool shuts down, and one that parked a
     * connection after the stop would start a second park thread, setting
     * running again under the thread being joined. */
    bool closed;
    pid_t owner;
    https_parked_t *head[2], *tail[2];
} g_park = {.lock = PTHREAD_MUTEX_INITIALIZER, .epoll_fd = -1};
 
static void park_unlink_locked(https_parked_t *p) {
    if (p->prev) p->prev->next = p->next;
    else g_park.head[p->cls] = p->next;
    if (p->next) p->next->prev = p->prev;
    else g_park.tail[p->cls] = p->prev;
    p->prev = p->next = NULL;
    p->linked = false;
}
 
static void park_resubmit(cwist_https_connection *conn, bool expired) {
    conn->park_expired = expired;
    https_pool_submit_conn(conn, conn->pool_ctx, conn->pool_handler, conn->pool_user_ctx);
}
 
static void *https_park_thread(void *arg) {
    (void)arg;
    struct epoll_event events[1024];
    while (1) {
        /* epoll_fd is (re)assigned under the lock by https_park_start_locked(),
         * so read it there together with running. */
        pthread_mutex_lock(&g_park.lock);
        bool running = g_park.running;
        int epfd = g_park.epoll_fd;
        pthread_mutex_unlock(&g_park.lock);
        if (!running) break;
 
        int n = epoll_wait(epfd, events, 1024, 200);
        for (int i = 0; i < n; i++) {
            https_parked_t *p = (https_parked_t *)events[i].data.ptr;
            pthread_mutex_lock(&g_park.lock);
            bool mine = p->linked;
            if (mine) park_unlink_locked(p);
            pthread_mutex_unlock(&g_park.lock);
            if (!mine) continue;
            epoll_ctl(epfd, EPOLL_CTL_DEL, p->conn->fd, NULL);
            /* Readable or hung up: the pool thread's read reports either. */
            park_resubmit(p->conn, false);
            cwist_free(p);
        }
 
        uint64_t now = cwist_https_now_ms();
        for (int cls = 0; cls < 2; cls++) {
            https_parked_t *expired = NULL;
            pthread_mutex_lock(&g_park.lock);
            while (g_park.head[cls] && g_park.head[cls]->deadline_ms <= now) {
                https_parked_t *p = g_park.head[cls];
                park_unlink_locked(p);
                p->next = expired;
                expired = p;
            }
            pthread_mutex_unlock(&g_park.lock);
            while (expired) {
                https_parked_t *p = expired;
                expired = p->next;
                epoll_ctl(epfd, EPOLL_CTL_DEL, p->conn->fd, NULL);
                if (cls == CWIST_HTTPS_PARK_HTTP1) {
                    https_park_teardown(p->conn);
                } else {
                    park_resubmit(p->conn, true);
                }
                cwist_free(p);
            }
        }
    }
    return NULL;
}
 
static bool https_park_start_locked(void) {
    pid_t pid = getpid();
    if (g_park.running && g_park.owner == pid) return true;
    if (g_park.closed && g_park.owner == pid) return false; /* pool is shutting down */
    /* A forked worker inherits the parent's lists and epoll fd but not its
     * thread; start its own set (the parent's parked connections are the
     * parent's to serve). */
    g_park.head[0] = g_park.head[1] = g_park.tail[0] = g_park.tail[1] = NULL;
    g_park.running = false;
    g_park.epoll_fd = epoll_create1(EPOLL_CLOEXEC);
    if (g_park.epoll_fd < 0) return false;
    g_park.running = true;
    g_park.owner = pid;
    if (pthread_create(&g_park.thread, NULL, https_park_thread, NULL) != 0) {
        g_park.running = false;
        close(g_park.epoll_fd);
        g_park.epoll_fd = -1;
        return false;
    }
    return true;
}
 
static bool https_park_enabled(void) {
    static int enabled = -1;
    if (enabled < 0) {
        const char *env = getenv("CWIST_HTTPS_PARK");
        enabled = !(env && (env[0] == '0' || strcmp(env, "false") == 0));
    }
    return enabled;
}
 
static void https_park_teardown(cwist_https_connection *conn) {
    if (cwist_full_gc_enabled()) cwist_conn_registry_untrack(conn);
    https_connection_teardown(conn);
}
 
bool cwist_https_park(cwist_https_connection *conn, uint64_t idle_ms, cwist_https_park_class cls) {
    if (!conn || conn->fd < 0 || !conn->pool_handler || !https_park_enabled()) return false;
    if (!atomic_load(&g_cwist_running)) return false;
    /* An idle connection with nothing buffered does not need its 16 KiB
     * read buffer while parked; cwist_https_receive_request() allocates a
     * fresh one when the connection is served again. Measured on the C100k
     * gate this was ~4 KB of resident memory per idle connection. Done
     * before the fd is armed below, while no other thread can see conn. */
    if (conn->read_buf && conn->buf_len == 0) {
        cwist_free(conn->read_buf);
        conn->read_buf = NULL;
    }
    https_parked_t *p = (https_parked_t *)cwist_alloc(sizeof(*p));
    if (!p) return false;
    memset(p, 0, sizeof(*p));
    if (cwist_full_gc_enabled()) cwist_gc_scope_disown(p);
    p->conn = conn;
    p->cls = cls == CWIST_HTTPS_PARK_HTTP2 ? CWIST_HTTPS_PARK_HTTP2 : CWIST_HTTPS_PARK_HTTP1;
    p->deadline_ms = cwist_https_now_ms() + idle_ms;
 
    pthread_mutex_lock(&g_park.lock);
    if (!https_park_start_locked()) {
        pthread_mutex_unlock(&g_park.lock);
        cwist_free(p);
        return false;
    }
    p->prev = g_park.tail[p->cls];
    if (p->prev) p->prev->next = p;
    else g_park.head[p->cls] = p;
    g_park.tail[p->cls] = p;
    p->linked = true;
    conn->park_expired = false;
    /* Register while holding the lock: once the fd is armed the park thread
     * may resubmit the connection, and it takes this lock first. */
    struct epoll_event ev = {.events = EPOLLIN | EPOLLRDHUP | EPOLLONESHOT, .data.ptr = p};
    if (epoll_ctl(g_park.epoll_fd, EPOLL_CTL_ADD, conn->fd, &ev) != 0) {
        park_unlink_locked(p);
        pthread_mutex_unlock(&g_park.lock);
        cwist_free(p);
        return false;
    }
    t_https_parked = true;
    pthread_mutex_unlock(&g_park.lock);
    return true;
}
 
static void https_park_reopen(void) {
    pthread_mutex_lock(&g_park.lock);
    g_park.closed = false;
    pthread_mutex_unlock(&g_park.lock);
}
 
static void https_park_stop(void) {
    pthread_mutex_lock(&g_park.lock);
    bool running = g_park.running && g_park.owner == getpid();
    g_park.running = false;
    if (running) g_park.closed = true;
    pthread_mutex_unlock(&g_park.lock);
    if (!running) return;
    pthread_join(g_park.thread, NULL);
    for (int cls = 0; cls < 2; cls++) {
        https_parked_t *p = g_park.head[cls];
        while (p) {
            https_parked_t *next = p->next;
            https_park_teardown(p->conn);
            cwist_free(p);
            p = next;
        }
        g_park.head[cls] = g_park.tail[cls] = NULL;
    }
    close(g_park.epoll_fd);
    g_park.epoll_fd = -1;
}
#else
bool cwist_https_park(cwist_https_connection *conn, uint64_t idle_ms, cwist_https_park_class cls) {
    (void)conn;
    (void)idle_ms;
    (void)cls;
    return false;
}
 
static void https_park_stop(void) {}
static void https_park_reopen(void) {}
#endif
 
bool cwist_https_conn_idle(cwist_https_connection *conn) {
    if (!conn || !conn->ssl || conn->buf_len > 0) return false;
    if (SSL_pending(conn->ssl) > 0) return false;
    struct pollfd pfd = {.fd = conn->fd, .events = POLLIN};
    return poll(&pfd, 1, 0) == 0;
}
 
/* Re-arm immediate ACKs on a connected socket.
 *
 * Why: TCP_QUICKACK is not persistent - the kernel re-enters delayed-ACK
 * (pingpong) mode during the TLS handshake's request/response segment
 * pattern.  A client that sends its TLS Finished and then holds the first
 * HTTP request behind Nagle (no TCP_NODELAY: stock ab, many OpenSSL-based
 * clients) waits for the ACK of the Finished; the server's delayed ACK
 * holds it for ~40 ms, adding a fixed ~40-50 ms to every connection and
 * capping churn throughput at ~20 handshakes/s/thread.  Re-arming around
 * the handshake keeps the ACK of the client's final flight immediate, so
 * the request arrives right behind the handshake instead of one delayed-ACK
 * window later.  Best-effort: non-Linux or failure is fine, this only
 * tunes ACK timing. */
static void cwist_tcp_quickack(int fd) {
#if defined(__linux__) && defined(TCP_QUICKACK)
    int one = 1;
    (void)setsockopt(fd, IPPROTO_TCP, TCP_QUICKACK, &one, sizeof(one));
#endif
}
 
uint64_t cwist_https_idle_timeout_ms(void) {
    static uint64_t cached = 0;
    if (!cached) {
        uint64_t v = CWIST_HTTP_TIMEOUT_MS;
        const char *env = getenv("CWIST_HTTPS_IDLE_TIMEOUT_MS");
        if (env && *env) {
            char *end = NULL;
            unsigned long long parsed = strtoull(env, &end, 10);
            if (end != env && *end == '\0' && parsed > 0) v = (uint64_t)parsed;
        }
        cached = v;
    }
    return cached;
}
 
static SSL *cwist_https_ssl_new(cwist_https_context *ctx) {
    if (!ctx) return NULL;
    pthread_rwlock_rdlock(&ctx->ctx_lock);
    SSL *ssl = ctx->ctx ? SSL_new(ctx->ctx) : NULL;
    pthread_rwlock_unlock(&ctx->ctx_lock);
    return ssl;
}
 
/* --- TLS handshake shepherd ------------------------------------------------
 * Why this exists: the pool parks one worker per connection for the whole
 * connection lifetime.  A synchronous in-worker handshake made that fatal
 * under churn — a client that delays its ClientHello by a few seconds
 * (routine for h2load at thousands of connections per process) parked a
 * worker for the full 30 s poll, the pool collapsed to ~tens of
 * handshakes/s, the accept loop fell behind, the kernel accept queue
 * overflowed, and overflowing handshakes were dropped silently, leaving
 * clients ESTABLISHED with no server-side socket ("phantom" connections).
 *
 * The shepherd owns every in-progress handshake: cwist_https_dispatch()
 * tries SSL_accept once on a non-blocking socket; incomplete handshakes are
 * parked in a private epoll set and retried by a shepherd thread, which also
 * reaps connections whose handshake stalls for CWIST_HTTPS_HANDSHAKE_TIMEOUT_MS
 * (any completed event round refreshes the budget — the deadline bounds
 * stalls, not queueing, so healthy clients under connect bursts are never
 * swept for waiting).  Only fully established sessions enter the worker pool.
 *
 * The shepherd is sharded (CWIST_HTTPS_HS_MAX_SHARDS threads, hashed by fd):
 * a single thread caps per-process handshake crypto at one core, and under
 * C100k-scale connect bursts the backlog outlived the handshake deadline,
 * resetting clients mid-connect ("errored" in h2load).  Shards carry
 * independent epoll sets, locks and pending lists, so throughput scales with
 * cores exactly like the pre-shepherd pool path did.
 *
 * The shepherd is epoll-based and therefore Linux-only.  On other platforms
 * cwist_https_dispatch() falls back to the legacy blocking pool path. */
 
#ifdef __linux__
 
typedef struct https_hs_pending {
    int fd;
    SSL *ssl;
    cwist_https_context *ctx;
    void (*handler)(cwist_https_connection *, void *);
    void *user_ctx;
    uint64_t deadline_ms;
    struct https_hs_pending *prev, *next;
    bool linked;
} https_hs_pending_t;
 
typedef struct https_hs_shard {
    int epoll_fd;
    int wakeup_rd;
    int wakeup_wr;
    pthread_t thread;
    /* Read lock-free by the shepherd loop and written by start/stop on other
     * threads, so it must stay atomic. */
    _Atomic bool running;
    pthread_mutex_t lock;
    /* Pending handshakes in deadline order: every entry is (re)queued at the
     * tail with now + CWIST_HTTPS_HANDSHAKE_TIMEOUT_MS, so the sweep only
     * looks at the head. */
    https_hs_pending_t *head, *tail;
    _Atomic uint32_t pending;
} https_hs_shard_t;
 
/* One shepherd thread per shard; the count is fixed at start (see below). */
#define CWIST_HTTPS_HS_MAX_SHARDS 16
 
static https_hs_shard_t g_hs_shards[CWIST_HTTPS_HS_MAX_SHARDS];
static long g_hs_shard_count = 0;
static pthread_mutex_t g_hs_start_lock = PTHREAD_MUTEX_INITIALIZER;
 
/* Both called with sh->lock held; O(1), so the shepherd's per-event and
 * sweep costs no longer grow with the number of pending handshakes. */
static void https_hs_link_locked(https_hs_shard_t *sh, https_hs_pending_t *p) {
    p->next = NULL;
    p->prev = sh->tail;
    if (sh->tail)
        sh->tail->next = p;
    else
        sh->head = p;
    sh->tail = p;
    p->linked = true;
    atomic_fetch_add_explicit(&sh->pending, 1, memory_order_release);
}
 
static void https_hs_unlink_locked(https_hs_shard_t *sh, https_hs_pending_t *p) {
    if (!p->linked) return;
    if (p->prev)
        p->prev->next = p->next;
    else
        sh->head = p->next;
    if (p->next)
        p->next->prev = p->prev;
    else
        sh->tail = p->prev;
    p->prev = p->next = NULL;
    p->linked = false;
    atomic_fetch_sub_explicit(&sh->pending, 1, memory_order_release);
}
 
static void https_hs_abort(https_hs_shard_t *sh, https_hs_pending_t *p) {
    if (sh->epoll_fd >= 0) epoll_ctl(sh->epoll_fd, EPOLL_CTL_DEL, p->fd, NULL);
    SSL_free(p->ssl);
    close(p->fd);
    cwist_free(p);
}
 
/* Move a completed handshake into the request worker pool. */
static void https_hs_complete(https_hs_shard_t *sh, https_hs_pending_t *p) {
    if (sh->epoll_fd >= 0) epoll_ctl(sh->epoll_fd, EPOLL_CTL_DEL, p->fd, NULL);
    /* The request phase (cwist_https_receive_request & friends) predates
     * non-blocking sockets; restore blocking mode before handing over. */
    int fl = fcntl(p->fd, F_GETFL, 0);
    if (fl >= 0) fcntl(p->fd, F_SETFL, fl & ~O_NONBLOCK);
    https_pool_submit_ready(p->fd, p->ssl, p->ctx, p->handler, p->user_ctx);
    cwist_free(p);
}
 
static void *https_hs_shepherd(void *arg) {
    https_hs_shard_t *sh = (https_hs_shard_t *)arg;
    struct epoll_event events[512];
    while (sh->running) {
        int n = epoll_wait(sh->epoll_fd, events, 512, 500);
        if (n < 0) {
            if (errno == EINTR) continue;
            break;
        }
        uint64_t now = cwist_https_now_ms();
        for (int i = 0; i < n; i++) {
            if (events[i].data.ptr == NULL) {
                /* wakeup pipe: drain so writers never block */
                char drain[256];
                while (read(sh->wakeup_rd, drain, sizeof(drain)) > 0) { /* discard */
                }
                continue;
            }
            https_hs_pending_t *p = events[i].data.ptr;
 
            /* Detach from the list first: every event path below either
             * re-arms (re-add) or finishes with p freed. */
            pthread_mutex_lock(&sh->lock);
            bool found = p->linked;
            https_hs_unlink_locked(sh, p);
            pthread_mutex_unlock(&sh->lock);
            if (!found) continue; /* already reaped by the sweeper */
 
            int rc = SSL_accept(p->ssl);
            cwist_tcp_quickack(p->fd);
            if (rc > 0) {
                https_hs_complete(sh, p);
                continue;
            }
            int ssl_err = SSL_get_error(p->ssl, rc);
            if ((ssl_err == SSL_ERROR_WANT_READ || ssl_err == SSL_ERROR_WANT_WRITE) &&
                now < p->deadline_ms) {
                /* The deadline bounds *stall*, not queueing: every completed
                 * event round means the client is alive and the handshake is
                 * progressing, so refresh the budget.  Under C100k-scale
                 * bursts a fixed from-accept deadline was consumed by queue
                 * delay alone and healthy handshakes were swept en masse. */
                p->deadline_ms = now + CWIST_HTTPS_HANDSHAKE_TIMEOUT_MS;
                struct epoll_event ev = {
                    .events = (ssl_err == SSL_ERROR_WANT_WRITE ? EPOLLOUT : EPOLLIN) | EPOLLRDHUP,
                    .data.ptr = p,
                };
                pthread_mutex_lock(&sh->lock);
                https_hs_link_locked(sh, p);
                pthread_mutex_unlock(&sh->lock);
                if (epoll_ctl(sh->epoll_fd, EPOLL_CTL_MOD, p->fd, &ev) != 0 &&
                    epoll_ctl(sh->epoll_fd, EPOLL_CTL_ADD, p->fd, &ev) != 0) {
                    pthread_mutex_lock(&sh->lock);
                    https_hs_unlink_locked(sh, p);
                    pthread_mutex_unlock(&sh->lock);
                    https_hs_abort(sh, p);
                }
                continue;
            }
            https_hs_abort(sh, p); /* hard failure or deadline exceeded */
        }
 
        /* Sweep expired handshakes that never became readable; the list is
         * in deadline order, so stop at the first live one. */
        uint64_t sweep_now = cwist_https_now_ms();
        pthread_mutex_lock(&sh->lock);
        while (sh->head && sweep_now >= sh->head->deadline_ms) {
            https_hs_pending_t *expired = sh->head;
            https_hs_unlink_locked(sh, expired);
            pthread_mutex_unlock(&sh->lock);
            https_hs_abort(sh, expired);
            pthread_mutex_lock(&sh->lock);
        }
        pthread_mutex_unlock(&sh->lock);
    }
    return NULL;
}
 
static void https_hs_shard_stop(https_hs_shard_t *sh) {
    pthread_mutex_lock(&sh->lock);
    if (!sh->running) {
        pthread_mutex_unlock(&sh->lock);
        return;
    }
    sh->running = false;
    if (sh->wakeup_wr >= 0) {
        char b = 1;
        if (write(sh->wakeup_wr, &b, 1) < 0) { /* shutdown path */
        }
    }
    pthread_mutex_unlock(&sh->lock);
    pthread_join(sh->thread, NULL);
 
    pthread_mutex_lock(&sh->lock);
    https_hs_pending_t *cur = sh->head;
    sh->head = sh->tail = NULL;
    pthread_mutex_unlock(&sh->lock);
    while (cur) {
        https_hs_pending_t *next = cur->next;
        SSL_free(cur->ssl);
        close(cur->fd);
        cwist_free(cur);
        cur = next;
    }
    if (sh->epoll_fd >= 0) close(sh->epoll_fd);
    if (sh->wakeup_rd >= 0) close(sh->wakeup_rd);
    if (sh->wakeup_wr >= 0) close(sh->wakeup_wr);
    sh->epoll_fd = -1;
    sh->wakeup_rd = sh->wakeup_wr = -1;
}
 
int https_hs_shepherd_start(void) {
    pthread_mutex_lock(&g_hs_start_lock);
    if (g_hs_shard_count > 0) {
        pthread_mutex_unlock(&g_hs_start_lock);
        return 0;
    }
    /* Spread handshakes across as many shepherd threads as this process has
     * request workers (bounded), matching the parallelism of the legacy
     * blocking pool path without parking request workers on handshakes. */
    long want = get_optimal_thread_count();
    if (want < 4) want = 4;
    if (want > CWIST_HTTPS_HS_MAX_SHARDS) want = CWIST_HTTPS_HS_MAX_SHARDS;
    /* CWIST_HTTPS_HS_SHARDS overrides the computed count for connect-burst
     * tuning; values outside [1, CWIST_HTTPS_HS_MAX_SHARDS] are ignored. */
    const char *env = getenv("CWIST_HTTPS_HS_SHARDS");
    if (env && *env) {
        char *end = NULL;
        long parsed = strtol(env, &end, 10);
        if (end != env && *end == '\0' && parsed >= 1 && parsed <= CWIST_HTTPS_HS_MAX_SHARDS)
            want = parsed;
    }
 
    long started = 0;
    for (long i = 0; i < want; i++) {
        https_hs_shard_t *sh = &g_hs_shards[i];
        memset(sh, 0, sizeof(*sh));
        sh->epoll_fd = -1;
        sh->wakeup_rd = sh->wakeup_wr = -1;
        pthread_mutex_init(&sh->lock, NULL);
        atomic_init(&sh->pending, 0);
 
        sh->epoll_fd = epoll_create1(0);
        if (sh->epoll_fd < 0) break;
        int pfd[2];
        if (pipe(pfd) != 0) {
            close(sh->epoll_fd);
            sh->epoll_fd = -1;
            break;
        }
        sh->wakeup_rd = pfd[0];
        sh->wakeup_wr = pfd[1];
        fcntl(sh->wakeup_rd, F_SETFL, O_NONBLOCK);
        fcntl(sh->wakeup_wr, F_SETFL, O_NONBLOCK);
        struct epoll_event ev = {.events = EPOLLIN, .data.ptr = NULL};
        epoll_ctl(sh->epoll_fd, EPOLL_CTL_ADD, sh->wakeup_rd, &ev);
        sh->running = true;
        if (pthread_create(&sh->thread, NULL, https_hs_shepherd, sh) != 0) {
            sh->running = false;
            close(sh->wakeup_rd);
            close(sh->wakeup_wr);
            close(sh->epoll_fd);
            sh->epoll_fd = -1;
            sh->wakeup_rd = sh->wakeup_wr = -1;
            break;
        }
        started++;
    }
    if (started == 0) {
        pthread_mutex_unlock(&g_hs_start_lock);
        return -1;
    }
    g_hs_shard_count = started;
    pthread_mutex_unlock(&g_hs_start_lock);
    return 0;
}
 
void https_hs_shepherd_stop(void) {
    pthread_mutex_lock(&g_hs_start_lock);
    long count = g_hs_shard_count;
    g_hs_shard_count = 0;
    pthread_mutex_unlock(&g_hs_start_lock);
    for (long i = 0; i < count; i++) {
        https_hs_shard_stop(&g_hs_shards[i]);
    }
}
 
long cwist_https_pending_handshakes(void) {
    long total = 0;
    for (long i = 0; i < g_hs_shard_count; i++) {
        total += atomic_load_explicit(&g_hs_shards[i].pending, memory_order_acquire);
    }
    return total;
}
 
void cwist_https_dispatch(int client_fd, cwist_https_context *ctx,
                          void (*handler)(cwist_https_connection *, void *), void *user_ctx) {
    if (!ctx || client_fd < 0) {
        if (client_fd >= 0) close(client_fd);
        return;
    }
    if (https_hs_shepherd_start() != 0) {
        /* No shepherd, no dispatch: fall back to the blocking pool path. */
        https_pool_submit(client_fd, ctx, handler, user_ctx);
        return;
    }
 
    int fl = fcntl(client_fd, F_GETFL, 0);
    if (fl >= 0) fcntl(client_fd, F_SETFL, fl | O_NONBLOCK);
 
    SSL *ssl = cwist_https_ssl_new(ctx);
    if (!ssl) {
        close(client_fd);
        return;
    }
    SSL_set_fd(ssl, client_fd);
 
    https_hs_pending_t *p = cwist_alloc(sizeof(*p));
    if (!p) {
        SSL_free(ssl);
        close(client_fd);
        return;
    }
    p->fd = client_fd;
    p->ssl = ssl;
    p->ctx = ctx;
    p->handler = handler;
    p->user_ctx = user_ctx;
    p->deadline_ms = cwist_https_now_ms() + CWIST_HTTPS_HANDSHAKE_TIMEOUT_MS;
 
    /* P2C (Power of Two Choices) Shard Selection:
     * Samples two shepherd shards and assigns to the one with lower pending count. */
    unsigned shard_idx = 0;
    if (g_hs_shard_count > 1) {
        _Atomic uint32_t pending_arr[CWIST_HTTPS_HS_MAX_SHARDS];
        for (long i = 0; i < g_hs_shard_count; i++) {
            atomic_init(&pending_arr[i],
                        atomic_load_explicit(&g_hs_shards[i].pending, memory_order_relaxed));
        }
        shard_idx =
            (unsigned)cwist_sched_p2c_select_worker(pending_arr, (uint32_t)g_hs_shard_count);
    }
 
    https_hs_shard_t *sh = &g_hs_shards[shard_idx];
    struct epoll_event ev = {
        .events = EPOLLIN | EPOLLOUT | EPOLLRDHUP | EPOLLET,
        .data.ptr = p,
    };
 
    pthread_mutex_lock(&sh->lock);
    https_hs_link_locked(sh, p);
    pthread_mutex_unlock(&sh->lock);
 
    if (epoll_ctl(sh->epoll_fd, EPOLL_CTL_ADD, client_fd, &ev) != 0) {
        pthread_mutex_lock(&sh->lock);
        https_hs_unlink_locked(sh, p);
        pthread_mutex_unlock(&sh->lock);
        SSL_free(ssl);
        close(client_fd);
        cwist_free(p);
        return;
    }
    /* Wake the shepherd so pending handshakes are processed promptly. */
    char b = 1;
    if (write(sh->wakeup_wr, &b, 1) < 0) { /* non-fatal */
    }
}
 
#else /* !__linux__ */
 
int https_hs_shepherd_start(void) {
    return -1;
}
void https_hs_shepherd_stop(void) {}
 
long cwist_https_pending_handshakes(void) {
    return 0;
}
 
/* Non-Linux fallback: no epoll shepherd, use the legacy blocking pool path. */
void cwist_https_dispatch(int client_fd, cwist_https_context *ctx,
                          void (*handler)(cwist_https_connection *, void *), void *user_ctx) {
    if (!ctx || client_fd < 0) {
        if (client_fd >= 0) close(client_fd);
        return;
    }
    https_pool_submit(client_fd, ctx, handler, user_ctx);
}
 
#endif /* __linux__ */
 
#define CWIST_ALPN_HTTP11 ((const unsigned char *)"\x08http/1.1")
#define CWIST_ALPN_H2_HTTP11 ((const unsigned char *)"\x02h2\x08http/1.1")
#define CWIST_ALPN_H3_H2_HTTP11 ((const unsigned char *)"\x02h3\x02h2\x08http/1.1")
#define CWIST_ALPN_HTTP11_LEN 9
#define CWIST_ALPN_H2_HTTP11_LEN 12
#define CWIST_ALPN_H3_H2_HTTP11_LEN 15
 
static cwist_error_t make_ssl_error(const char *msg) {
    cwist_error_t err = make_error(CWIST_ERR_JSON);
    err.error.err_json = cJSON_CreateObject();
 
    unsigned long ssl_err = ERR_get_error();
    char buf[256];
    ERR_error_string_n(ssl_err, buf, sizeof(buf));
 
    cJSON_AddStringToObject(err.error.err_json, "module", "https");
    cJSON_AddStringToObject(err.error.err_json, "message", msg);
    cJSON_AddStringToObject(err.error.err_json, "openssl_error", buf);
 
    return err;
}
 
static void cwist_https_apply_base_tls_defaults(SSL_CTX *ssl_ctx) {
    SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_COMPRESSION);
#ifdef SSL_OP_NO_RENEGOTIATION
    SSL_CTX_set_options(ssl_ctx, SSL_OP_NO_RENEGOTIATION);
#endif
    SSL_CTX_set_mode(ssl_ctx, SSL_MODE_AUTO_RETRY);
 
    /* Session resumption. TLS 1.3 tickets are stateless and already on by
     * default; this additionally enables the server-side session cache for
     * TLS 1.2 session IDs and sets the session-id context without which
     * OpenSSL refuses to resume cached sessions. Combined with HTTP/1.1
     * keep-alive this removes most full (asymmetric) handshakes. */
    SSL_CTX_set_session_cache_mode(ssl_ctx, SSL_SESS_CACHE_SERVER);
    SSL_CTX_sess_set_cache_size(ssl_ctx, cwist_tls_sess_cache_size());
    SSL_CTX_set_session_id_context(ssl_ctx, (const unsigned char *)"cwist-v1", 8);
    SSL_CTX_set_timeout(ssl_ctx, 300);
}
 
static cwist_error_t cwist_https_apply_http2_tls_profile(SSL_CTX *ssl_ctx) {
    cwist_error_t err = make_error(CWIST_ERR_INT16);
    if (SSL_CTX_set_cipher_list(ssl_ctx, "ECDHE-ECDSA-AES128-GCM-SHA256:"
                                         "ECDHE-RSA-AES128-GCM-SHA256:"
                                         "ECDHE-ECDSA-AES256-GCM-SHA384:"
                                         "ECDHE-RSA-AES256-GCM-SHA384:"
                                         "ECDHE-ECDSA-CHACHA20-POLY1305:"
                                         "ECDHE-RSA-CHACHA20-POLY1305") != 1) {
        return make_ssl_error("Unable to apply HTTP/2-compatible TLS 1.2 cipher profile");
    }
#ifdef SSL_CTX_set_ciphersuites
    if (SSL_CTX_set_ciphersuites(ssl_ctx, "TLS_AES_128_GCM_SHA256:"
                                          "TLS_AES_256_GCM_SHA384:"
                                          "TLS_CHACHA20_POLY1305_SHA256") != 1) {
        return make_ssl_error("Unable to apply HTTP/2-compatible TLS 1.3 cipher suites");
    }
#endif
    SSL_CTX_set_options(ssl_ctx, SSL_OP_CIPHER_SERVER_PREFERENCE);
    err.error.err_i16 = 0;
    return err;
}
 
static int cwist_https_alpn_select_cb(SSL *ssl, const unsigned char **out, unsigned char *outlen,
                                      const unsigned char *in, unsigned int inlen, void *arg) {
    (void)ssl;
    const cwist_https_context *hctx = (const cwist_https_context *)arg;
    bool enable_http2 = hctx && hctx->http2_enabled;
 
    const unsigned char *supported;
    unsigned int supported_len;
 
    if (enable_http2) {
        supported = CWIST_ALPN_H2_HTTP11;
        supported_len = CWIST_ALPN_H2_HTTP11_LEN;
    } else {
        supported = CWIST_ALPN_HTTP11;
        supported_len = CWIST_ALPN_HTTP11_LEN;
    }
 
    if (SSL_select_next_proto((unsigned char **)out, outlen, supported, supported_len, in, inlen) ==
        OPENSSL_NPN_NEGOTIATED) {
        return SSL_TLSEXT_ERR_OK;
    }
 
    return SSL_TLSEXT_ERR_NOACK;
}
 
/* --- Context Management --- */
 
/* --- Shared session ticket key -------------------------------------------
 * cwist_app_listen() forks one worker per CPU core. BoringSSL/OpenSSL
 * generate their internal session-ticket keys lazily on first use, i.e.
 * AFTER the fork, so every worker would mint its own key and tickets issued
 * by one worker could never be resumed on another (SO_REUSEPORT scatters
 * connections across workers). Instead we generate one random key at
 * context creation (in the parent, pre-fork) and install an explicit
 * ticket-key callback; every forked worker inherits identical key material
 * and tickets resume regardless of which worker accepts the connection.
 *
 * Security trade-off — process-lifetime STEK (tracked in issue #98):
 *
 * The key lives for the process lifetime (no rotation), the same design
 * nginx uses with a static `ssl_session_ticket_key` file.  That is a
 * deliberate choice: rotation in a pre-fork model requires either a shared
 * mmap region with locking (cross-process visibility) or a control socket
 * from a parent/daemon process — significant complexity for a server that
 * is typically restarted nightly anyway.
 *
 * Actual exposure with the current codebase:
 *
 * 1. 0-RTT / early-data replay does NOT apply here.  CWIST never calls
 *    SSL_CTX_set_max_early_data() or equivalent, so early data is
 *    disabled at the library level and TLS 1.3 0-RTT is never offered.
 *
 * 2. What remains is the blast-radius argument: if the STEK leaks (memory
 *    disclosure, core dump, cold-boot) an attacker can decrypt any
 *    session ticket issued by this process for the rest of its uptime.
 *    Rotation would cap the valid window to the rotation interval.
 *
 * Mitigations short of full rotation:
 *   - Enable perfect forward secrecy by preferring ECDHE cipher suites
 *     (already the default here via BoringSSL's suite ordering) so that
 *     bulk traffic keys are independent of the STEK even if the STEK leaks.
 *   - Restart the server on a short cadence (e.g. daily) if the uptime
 *     window is a concern for your threat model; a new process generates a
 *     fresh STEK via RAND_bytes() in cwist_tls_setup_shared_ticket_key().
 *   - Disable ticket resumption entirely (SSL_CTX_set_options with
 *     SSL_OP_NO_TICKET) to opt into full handshakes on every connection.
 *
 * If in-process periodic rotation is ever added, the right approach is:
 *   a) Generate a new key on a timer (e.g. every hour).
 *   b) Keep the *previous* key for one rotation interval so in-flight
 *      tickets from the overlap window can still decrypt.
 *   c) Use an atomic pointer swap or reader-writer lock to avoid a
 *      race between the rotate timer and cwist_tls_ticket_key_cb().
 *
 * Certificate hot reload (cwist_https_reload_context()) does NOT rotate this
 * key: the reloaded SSL_CTX carries the same STEK so session resumption
 * survives the swap. See docs/decisions/0002-tls-hot-reload-stek.md.
 * ------------------------------------------------------------------------- */
typedef struct cwist_tls_ticket_key {
    unsigned char name[16];     /* key_name sent in the ticket */
    unsigned char aes_key[32];  /* AES-256-CBC encryption key  */
    unsigned char hmac_key[32]; /* HMAC-SHA-256 MAC key        */
} cwist_tls_ticket_key;
 
static int g_ticket_key_ex_data_idx = -1;
static pthread_once_t g_ticket_key_ex_data_once = PTHREAD_ONCE_INIT;
 
static void cwist_tls_ticket_key_ex_data_init(void) {
    g_ticket_key_ex_data_idx = SSL_CTX_get_ex_new_index(0, NULL, NULL, NULL, NULL);
}
 
static int cwist_tls_ticket_key_cb(SSL *ssl, uint8_t *key_name, uint8_t *iv, EVP_CIPHER_CTX *ectx,
                                   HMAC_CTX *hctx, int encrypt) {
    SSL_CTX *ssl_ctx = ssl ? SSL_get_SSL_CTX(ssl) : NULL;
    const cwist_tls_ticket_key *key = NULL;
    if (ssl_ctx && g_ticket_key_ex_data_idx >= 0) {
        key = (const cwist_tls_ticket_key *)SSL_CTX_get_ex_data(ssl_ctx, g_ticket_key_ex_data_idx);
    }
    if (!key) return 0;
 
    if (encrypt) {
        memcpy(key_name, key->name, sizeof(key->name));
        if (RAND_bytes(iv, 16) != 1) return 0;
        if (EVP_EncryptInit_ex(ectx, EVP_aes_256_cbc(), NULL, key->aes_key, iv) != 1) return 0;
        if (HMAC_Init_ex(hctx, key->hmac_key, sizeof(key->hmac_key), EVP_sha256(), NULL) != 1)
            return 0;
        return 1;
    }
 
    if (memcmp(key_name, key->name, sizeof(key->name)) != 0) {
        return 0; /* Unknown key: decline the ticket, do a full handshake. */
    }
    if (EVP_DecryptInit_ex(ectx, EVP_aes_256_cbc(), NULL, key->aes_key, iv) != 1) return 0;
    if (HMAC_Init_ex(hctx, key->hmac_key, sizeof(key->hmac_key), EVP_sha256(), NULL) != 1) return 0;
    return 1;
}
 
static void cwist_tls_setup_shared_ticket_key(SSL_CTX *ssl_ctx, void **out_key) {
    *out_key = NULL;
    pthread_once(&g_ticket_key_ex_data_once, cwist_tls_ticket_key_ex_data_init);
    if (g_ticket_key_ex_data_idx < 0) return;
 
    cwist_tls_ticket_key *key = (cwist_tls_ticket_key *)cwist_alloc(sizeof(*key));
    if (!key) return;
    if (RAND_bytes((uint8_t *)key, sizeof(*key)) != 1) {
        cwist_free(key);
        return; /* Fall back to the library-internal (per-worker) keys. */
    }
    SSL_CTX_set_ex_data(ssl_ctx, g_ticket_key_ex_data_idx, key);
    SSL_CTX_set_tlsext_ticket_key_cb(ssl_ctx, cwist_tls_ticket_key_cb);
    *out_key = key;
}
 
cwist_error_t cwist_https_init_context(cwist_https_context **ctx, const char *cert_path,
                                       const char *key_path) {
    return cwist_https_init_context_with_options(ctx, cert_path, key_path, NULL, NULL);
}
 
static cwist_error_t cwist_https_build_ssl_ctx(SSL_CTX **out, const char *cert_path,
                                               const char *key_path,
                                               const cwist_https_options *options,
                                               cwist_app *app) {
    cwist_error_t err = make_error(CWIST_ERR_INT16);
    bool enable_http3 = options && options->enable_http3;
    bool enable_http2 = options && (options->enable_http2 || enable_http3);
 
    // Initialize OpenSSL
    SSL_load_error_strings();
    OpenSSL_add_ssl_algorithms();
 
    const SSL_METHOD *method = TLS_server_method();
    SSL_CTX *ssl_ctx = SSL_CTX_new(method);
    if (!ssl_ctx) {
        return make_ssl_error("Unable to create SSL context");
    }
 
    if (SSL_CTX_set_min_proto_version(ssl_ctx, TLS1_2_VERSION) != 1) {
        SSL_CTX_free(ssl_ctx);
        return make_ssl_error("Unable to enforce TLS minimum version");
    }
    cwist_https_apply_base_tls_defaults(ssl_ctx);
 
    if (!cwist_tls_apply_pqc_layer(app, ssl_ctx)) {
        SSL_CTX_free(ssl_ctx);
        return make_ssl_error("PQC layer configuration failed");
    }
 
    if (enable_http2) {
        err = cwist_https_apply_http2_tls_profile(ssl_ctx);
        if (!cwist_error_is_ok(&err)) {
            SSL_CTX_free(ssl_ctx);
            return err;
        }
    }
 
    // Load Cert and Key
    if (SSL_CTX_use_certificate_chain_file(ssl_ctx, cert_path) <= 0) {
        SSL_CTX_free(ssl_ctx);
        return make_ssl_error("Unable to load certificate");
    }
 
    if (cwist_tls_autoload_intermediates(ssl_ctx) < 0) {
        SSL_CTX_free(ssl_ctx);
        return make_ssl_error("Unable to complete certificate chain");
    }
 
    if (SSL_CTX_use_PrivateKey_file(ssl_ctx, key_path, SSL_FILETYPE_PEM) <= 0) {
        SSL_CTX_free(ssl_ctx);
        return make_ssl_error("Unable to load private key");
    }
 
    // Verify key matches cert
    if (!SSL_CTX_check_private_key(ssl_ctx)) {
        SSL_CTX_free(ssl_ctx);
        return make_ssl_error("Private key does not match certificate");
    }
 
    *out = ssl_ctx;
    err.error.err_i16 = 0;
    return err;
}
 
cwist_error_t cwist_https_init_context_with_options(cwist_https_context **ctx,
                                                    const char *cert_path, const char *key_path,
                                                    const cwist_https_options *options,
                                                    cwist_app *app) {
    cwist_error_t err = make_error(CWIST_ERR_INT16);
    bool enable_http3 = options && options->enable_http3;
    bool enable_http2 = options && (options->enable_http2 || enable_http3);
 
    if (!ctx || !cert_path || !key_path) {
        err.error.err_i16 = -1;
        return err;
    }
 
    SSL_CTX *ssl_ctx = NULL;
    err = cwist_https_build_ssl_ctx(&ssl_ctx, cert_path, key_path, options, app);
    if (!cwist_error_is_ok(&err)) {
        return err;
    }
 
    *ctx = (cwist_https_context *)cwist_alloc(sizeof(cwist_https_context));
    if (!*ctx) {
        SSL_CTX_free(ssl_ctx);
        err.error.err_i16 = -1;
        return err;
    }
    (*ctx)->ctx = ssl_ctx;
    (*ctx)->http2_enabled = enable_http2;
    (*ctx)->http3_enabled = enable_http3;
    (*ctx)->ticket_key = NULL;
    pthread_rwlock_init(&(*ctx)->ctx_lock, NULL);
 
    /* Pre-fork shared ticket key so every forked worker can resume tickets
     * issued by any other worker. Must run before workers are forked. */
    cwist_tls_setup_shared_ticket_key(ssl_ctx, &(*ctx)->ticket_key);
 
    SSL_CTX_set_alpn_select_cb(ssl_ctx, cwist_https_alpn_select_cb, (void *)*ctx);
 
    err.error.err_i16 = 0; // Success
    return err;
}
 
cwist_error_t cwist_https_reload_context(cwist_https_context *ctx, const char *cert_path,
                                         const char *key_path, cwist_app *app) {
    cwist_error_t err = make_error(CWIST_ERR_INT16);
    if (!ctx || !cert_path || !key_path) {
        err.error.err_i16 = -1;
        return err;
    }
 
    cwist_https_options options = {.enable_http2 = ctx->http2_enabled,
                                   .enable_http3 = ctx->http3_enabled};
    SSL_CTX *new_ctx = NULL;
    err = cwist_https_build_ssl_ctx(&new_ctx, cert_path, key_path, &options, app);
    if (!cwist_error_is_ok(&err)) {
        return err;
    }
 
    /* Carry the shared STEK over: the wrapper owns it and outlives every
     * SSL_CTX it is installed on (it is freed only by
     * cwist_https_destroy_context(), after the pool is drained). The once
     * gate runs here because the reloading thread may not be the one that
     * created the context. */
    pthread_once(&g_ticket_key_ex_data_once, cwist_tls_ticket_key_ex_data_init);
    if (ctx->ticket_key && g_ticket_key_ex_data_idx >= 0) {
        SSL_CTX_set_ex_data(new_ctx, g_ticket_key_ex_data_idx, ctx->ticket_key);
        SSL_CTX_set_tlsext_ticket_key_cb(new_ctx, cwist_tls_ticket_key_cb);
    } else if (!ctx->ticket_key) {
        /* No shared key was ever armed: do it now so the reloaded context
         * matches a fresh one. */
        cwist_tls_setup_shared_ticket_key(new_ctx, &ctx->ticket_key);
    }
 
    SSL_CTX_set_alpn_select_cb(new_ctx, cwist_https_alpn_select_cb, (void *)ctx);
 
    pthread_rwlock_wrlock(&ctx->ctx_lock);
    SSL_CTX *old_ctx = ctx->ctx;
    ctx->ctx = new_ctx;
    pthread_rwlock_unlock(&ctx->ctx_lock);
 
    /* Drops only the listener's reference; in-flight handshakes and
     * established connections hold their own. */
    if (old_ctx) SSL_CTX_free(old_ctx);
 
    err.error.err_i16 = 0;
    return err;
}
 
void cwist_https_destroy_context(cwist_https_context *ctx) {
    if (ctx) {
        if (ctx->ctx) {
            SSL_CTX_free(ctx->ctx);
        }
        pthread_rwlock_destroy(&ctx->ctx_lock);
        if (ctx->ticket_key) {
            /* Scrub key material before releasing it. */
            OPENSSL_cleanse(ctx->ticket_key, sizeof(cwist_tls_ticket_key));
            cwist_free(ctx->ticket_key);
        }
        cwist_free(ctx);
        /* EVP_cleanup() intentionally not called: it is deprecated since
         * OpenSSL 1.1.0 and tears down global state other code may use. */
    }
}
 
/* --- TLS observability counters ------------------------------------------
 * Owned here (not in the metrics registry) so the hot path never depends
 * on the sys/metrics layer; metrics.c mirrors them into the Prometheus
 * exposition on load/render, same pattern as
 * cwist_http_continuation_shed_count() in http.c. */
static _Atomic long g_tls_handshakes_total;
static _Atomic long g_tls_handshakes_resumed;
static _Atomic long g_tls_handshakes_tls12;
static _Atomic long g_tls_handshakes_tls13;
static _Atomic long g_tls_ciphers_aes128_gcm;
static _Atomic long g_tls_ciphers_aes256_gcm;
static _Atomic long g_tls_ciphers_chacha20;
static _Atomic long g_tls_ciphers_other;
static _Atomic long g_tls_connections_active;
 
long cwist_https_tls_handshakes_total(void) {
    return atomic_load_explicit(&g_tls_handshakes_total, memory_order_relaxed);
}
 
long cwist_https_tls_handshakes_resumed_total(void) {
    return atomic_load_explicit(&g_tls_handshakes_resumed, memory_order_relaxed);
}
 
long cwist_https_tls_handshakes_tls12_total(void) {
    return atomic_load_explicit(&g_tls_handshakes_tls12, memory_order_relaxed);
}
 
long cwist_https_tls_handshakes_tls13_total(void) {
    return atomic_load_explicit(&g_tls_handshakes_tls13, memory_order_relaxed);
}
 
long cwist_https_tls_ciphers_aes128_gcm_total(void) {
    return atomic_load_explicit(&g_tls_ciphers_aes128_gcm, memory_order_relaxed);
}
 
long cwist_https_tls_ciphers_aes256_gcm_total(void) {
    return atomic_load_explicit(&g_tls_ciphers_aes256_gcm, memory_order_relaxed);
}
 
long cwist_https_tls_ciphers_chacha20_total(void) {
    return atomic_load_explicit(&g_tls_ciphers_chacha20, memory_order_relaxed);
}
 
long cwist_https_tls_ciphers_other_total(void) {
    return atomic_load_explicit(&g_tls_ciphers_other, memory_order_relaxed);
}
 
long cwist_https_tls_connections_active(void) {
    return atomic_load_explicit(&g_tls_connections_active, memory_order_relaxed);
}
 
/* Record a freshly established TLS session: version/cipher buckets and
 * resumption. Called from https_wrap_established only, which both the
 * in-worker handshake path (cwist_https_accept) and the shepherd's
 * pre-handshaked dispatch path funnel through. */
static void https_tls_record_handshake(SSL *ssl) {
    atomic_fetch_add_explicit(&g_tls_handshakes_total, 1, memory_order_relaxed);
    if (SSL_session_reused(ssl))
        atomic_fetch_add_explicit(&g_tls_handshakes_resumed, 1, memory_order_relaxed);
 
    switch (SSL_version(ssl)) {
        case TLS1_2_VERSION:
            atomic_fetch_add_explicit(&g_tls_handshakes_tls12, 1, memory_order_relaxed);
            break;
        case TLS1_3_VERSION:
            atomic_fetch_add_explicit(&g_tls_handshakes_tls13, 1, memory_order_relaxed);
            break;
        default: break;
    }
 
    const char *cipher = SSL_get_cipher_name(ssl);
    if (cipher) {
        if (strcmp(cipher, "TLS_AES_128_GCM_SHA256") == 0)
            atomic_fetch_add_explicit(&g_tls_ciphers_aes128_gcm, 1, memory_order_relaxed);
        else if (strcmp(cipher, "TLS_AES_256_GCM_SHA384") == 0)
            atomic_fetch_add_explicit(&g_tls_ciphers_aes256_gcm, 1, memory_order_relaxed);
        else if (strcmp(cipher, "TLS_CHACHA20_POLY1305_SHA256") == 0)
            atomic_fetch_add_explicit(&g_tls_ciphers_chacha20, 1, memory_order_relaxed);
        else
            atomic_fetch_add_explicit(&g_tls_ciphers_other, 1, memory_order_relaxed);
    }
}
 
/* Build the connection wrapper around an already-established TLS session.
 * Shared by the blocking cwist_https_accept path and the shepherd's
 * non-blocking dispatch path. */
static cwist_error_t https_wrap_established(cwist_https_context *ctx, int client_fd, SSL *ssl,
                                            cwist_https_connection **conn) {
    cwist_error_t err = make_error(CWIST_ERR_INT16);
 
    *conn = (cwist_https_connection *)cwist_alloc(sizeof(cwist_https_connection));
    if (!*conn) {
        err.error.err_i16 = -1;
        return err;
    }
 
    memset(*conn, 0, sizeof(**conn));
    (*conn)->fd = client_fd;
    (*conn)->ssl = ssl;
    (*conn)->read_buf = cwist_alloc(CWIST_HTTP_READ_BUFFER_SIZE);
    if (!(*conn)->read_buf) {
        cwist_free(*conn);
        *conn = NULL;
        err.error.err_i16 = -1;
        return err;
    }
    (*conn)->buf_len = 0;
    (*conn)->read_buf[0] = '\0';
    (*conn)->negotiated_http2 = false;
    (*conn)->negotiated_protocol = CWIST_HTTPS_PROTOCOL_HTTP11;
    (*conn)->http3_enabled = ctx->http3_enabled;
    (*conn)->deferred = false;
 
    if (cwist_full_gc_enabled()) {
        /* *conn and its read_buf were just handed to the connection
         * registry's close_fn (below) as their sole eventual releaser.
         * They must NOT also sit on this thread's generic cwist_alloc
         * pending-sweep list: that list's own thread-exit destructor runs
         * independently of (and in unspecified order relative to) the
         * connection registry's destructor, so if it fired first it would
         * cwist_ebr_free() *conn out from under the registry's still-
         * pending entry -- the registry's sweep would then read a freed
         * conn->ssl. Disowning here hands sole ownership to the registry,
         * exactly the cross-list handoff cwist_gc_scope_disown() exists
         * for (see io_queue.c's job-handoff use of the same call). */
        cwist_gc_scope_disown(*conn);
        cwist_gc_scope_disown((*conn)->read_buf);
        /* Full-GC's safety net: if this connection is never explicitly
         * closed (worker crashes mid-request, an error path forgets), the
         * owning thread's exit -- or, failing that, process exit -- still
         * closes it instead of leaking the fd/TLS session. Normal explicit
         * close (cwist_https_close_connection) untracks first, so this is
         * a no-op on the common path. Registering here (rather than only
         * in cwist_https_accept) covers the shepherd's pre-handshaked
         * dispatch path too, which calls this function directly. */
        cwist_conn_registry_track(*conn, https_close_conn_cb);
    }
 
    const unsigned char *alpn = NULL;
    unsigned int alpn_len = 0;
    SSL_get0_alpn_selected(ssl, &alpn, &alpn_len);
    if (alpn && alpn_len == 2 && memcmp(alpn, "h2", 2) == 0) {
        (*conn)->negotiated_http2 = true;
        (*conn)->negotiated_protocol = CWIST_HTTPS_PROTOCOL_HTTP2;
    }
    /* h3 is QUIC-only and never negotiated over TCP TLS */
 
    /* The handshake's final client flight was just ACKed; stay in quickack
     * mode so the request that follows (held back by a Nagle-bound client)
     * is ACKed immediately and the connection does not stall one
     * delayed-ACK window before the request phase even starts. */
    cwist_tcp_quickack(client_fd);
 
    /* Both handshake paths funnel through here, so this is the single
     * choke point for TLS observability (issue #306). */
    https_tls_record_handshake(ssl);
    atomic_fetch_add_explicit(&g_tls_connections_active, 1, memory_order_relaxed);
 
    err.error.err_i16 = 0;
    return err;
}
 
static void https_close_conn_cb(void *handle) {
    https_connection_teardown((cwist_https_connection *)handle);
}
 
cwist_error_t cwist_https_accept(cwist_https_context *ctx, int client_fd,
                                 cwist_https_connection **conn) {
    cwist_error_t err = make_error(CWIST_ERR_INT16);
 
    if (!ctx || client_fd < 0) {
        err.error.err_i16 = -1;
        return err;
    }
 
    SSL *ssl = cwist_https_ssl_new(ctx);
    if (!ssl) {
        return make_ssl_error("Failed to create SSL structure");
    }
 
    SSL_set_fd(ssl, client_fd);
    cwist_tcp_quickack(client_fd);
 
    /* Bound the whole handshake so a client dribbling bytes cannot pin a
     * pool worker forever. */
    uint64_t handshake_deadline = cwist_https_now_ms() + CWIST_HTTPS_HANDSHAKE_TIMEOUT_MS;
    int rc;
    while ((rc = SSL_accept(ssl)) <= 0) {
        /* Keep immediate ACKs armed: the kernel drops out of quickack mode
         * as the handshake segments flow, and the delayed ACK of the
         * client's Finished would hold a Nagle-bound request by ~40 ms. */
        cwist_tcp_quickack(client_fd);
        int ssl_err = SSL_get_error(ssl, rc);
        if (ssl_err == SSL_ERROR_WANT_READ || ssl_err == SSL_ERROR_WANT_WRITE) {
            uint64_t now = cwist_https_now_ms();
            if (now >= handshake_deadline) {
                cwist_error_t err_obj = make_ssl_error("SSL handshake timed out");
                SSL_free(ssl);
                return err_obj;
            }
            int wait_ms = CWIST_HTTP_TIMEOUT_MS;
            uint64_t remaining = handshake_deadline - now;
            if (remaining < (uint64_t)wait_ms) wait_ms = (int)remaining;
            if (cwist_ssl_wait(client_fd, ssl_err, wait_ms) != 0) {
                cwist_error_t err_obj = make_ssl_error("SSL handshake timed out or socket error");
                SSL_free(ssl);
                return err_obj;
            }
            continue;
        }
        cwist_error_t err_obj = make_ssl_error("SSL handshake failed");
        SSL_free(ssl);
        return err_obj;
    }
 
    cwist_error_t wrap_err = https_wrap_established(ctx, client_fd, ssl, conn);
    if (!cwist_error_is_ok(&wrap_err)) {
        SSL_free(ssl);
        return wrap_err;
    }
    return wrap_err;
}
 
bool cwist_https_connection_uses_http2(const cwist_https_connection *conn) {
    return conn && conn->negotiated_protocol == CWIST_HTTPS_PROTOCOL_HTTP2;
}
 
cwist_https_protocol cwist_https_connection_protocol(const cwist_https_connection *conn) {
    if (!conn) return CWIST_HTTPS_PROTOCOL_NONE;
    return conn->negotiated_protocol;
}
 
static void https_connection_teardown(cwist_https_connection *conn) {
    if (conn) {
        /* Balances the increment at wrap time; teardown is the single exit
         * point for every established connection (public close and the
         * full-GC registry sweep alike). */
        atomic_fetch_sub_explicit(&g_tls_connections_active, 1, memory_order_relaxed);
        if (conn->proto_state && conn->proto_state_free) {
            conn->proto_state_free(conn->proto_state);
            conn->proto_state = NULL;
        }
        if (conn->ssl) {
            SSL_shutdown(conn->ssl);
            SSL_free(conn->ssl);
        }
        if (conn->fd >= 0) {
            /* close() on a socket with unread receive-queue data makes the
             * kernel answer with RST instead of a graceful FIN. TLS clients
             * routinely send their close_notify (or a pipelined next request)
             * that the request path never consumed, so under connection churn
             * nearly every teardown became an abortive close: the peer's
             * kernel flushes queued response bytes on the RST (the "read
             * error" burst load clients report) and the socket never settles
             * through TIME_WAIT normally. Drain whatever is pending before
             * closing; the socket is about to be destroyed anyway. */
            char drain[2048];
            ssize_t n;
            int guard = 16;
            while (guard-- > 0 && (n = recv(conn->fd, drain, sizeof(drain), MSG_DONTWAIT)) > 0) {
                /* discard */
            }
            (void)n;
            close(conn->fd);
        }
        cwist_free(conn->read_buf);
        cwist_free(conn);
    }
}
 
void cwist_https_close_connection(cwist_https_connection *conn) {
    if (conn && cwist_full_gc_enabled()) {
        cwist_conn_registry_untrack(conn);
    }
    https_connection_teardown(conn);
}
 
cwist_http_request *cwist_https_receive_request(cwist_https_connection *conn) {
    if (!conn || !conn->ssl) return NULL;
    if (!conn->read_buf) {
        /* Released when the connection was parked (cwist_https_park()). */
        conn->read_buf = cwist_alloc(CWIST_HTTP_READ_BUFFER_SIZE);
        if (!conn->read_buf) return NULL;
        /* Same ownership as at wrap time: the conn's teardown frees it, so
         * it must not sit on this thread's full-GC sweep list. */
        if (cwist_full_gc_enabled()) cwist_gc_scope_disown(conn->read_buf);
        conn->buf_len = 0;
        conn->read_buf[0] = '\0';
    }
 
    size_t total_received = conn->buf_len;
    char *header_end = NULL;
 
    /* Total deadline for assembling the request headers. */
    uint64_t headers_deadline = cwist_https_now_ms() + CWIST_HTTP_HEADERS_TIMEOUT_MS;
 
    /* Rescanning from offset 0 on every read is O(n^2) over dribbled
     * headers; resume 3 bytes back so a terminator split across two reads
     * is still found. */
    size_t scan_from = 0;
    bool deadline_grace_used = false;
 
    while (1) {
        header_end = strstr(conn->read_buf + scan_from, "\r\n\r\n");
        if (header_end) break;
        scan_from = total_received > 3 ? total_received - 3 : 0;
 
        if (total_received >= CWIST_HTTP_READ_BUFFER_SIZE - 1) {
            return NULL;
        }
 
        if (SSL_pending(conn->ssl) == 0) {
            uint64_t now = cwist_https_now_ms();
            if (now >= headers_deadline) {
                /* A request may have landed just as the idle deadline
                 * expired; dropping it here makes strict clients surface a
                 * network protocol error where lenient ones silently retry.
                 * Give the wire one non-blocking chance before closing.
                 * Bounded to a single retry so a dribbling peer cannot pin
                 * the worker past the deadline. */
                bool readable = false;
                if (!deadline_grace_used) {
                    deadline_grace_used = true;
                    struct pollfd gfd = {.fd = conn->fd, .events = POLLIN};
                    readable = (poll(&gfd, 1, 0) > 0 && (gfd.revents & POLLIN));
                }
                if (!readable) {
                    return NULL;
                }
            } else {
                int wait_ms = CWIST_HTTP_TIMEOUT_MS;
                uint64_t remaining = headers_deadline - now;
                if (remaining < (uint64_t)wait_ms) wait_ms = (int)remaining;
 
                struct pollfd pfd = {.fd = conn->fd, .events = POLLIN};
                int pret = poll(&pfd, 1, wait_ms);
                if (pret <= 0) {
                    return NULL;
                }
            }
        }
 
        int bytes = SSL_read(conn->ssl, conn->read_buf + total_received,
                             (int)(CWIST_HTTP_READ_BUFFER_SIZE - 1 - total_received));
        if (bytes <= 0) {
            int ssl_err = SSL_get_error(conn->ssl, bytes);
            if (ssl_err == SSL_ERROR_WANT_READ || ssl_err == SSL_ERROR_WANT_WRITE) {
                if (cwist_ssl_wait(conn->fd, ssl_err, CWIST_HTTP_TIMEOUT_MS) != 0) return NULL;
                continue;
            }
            return NULL;
        }
 
        total_received += (size_t)bytes;
        conn->read_buf[total_received] = '\0';
    }
 
    cwist_http_request *req = cwist_http_parse_request(conn->read_buf);
    if (!req) return NULL;
 
    req->client_fd = conn->fd;
    req->https_conn = conn;
 
    size_t header_len = (header_end + 4) - conn->read_buf;
    size_t body_received = total_received - header_len;
 
    if (req->content_length > 0) {
        if (req->content_length > CWIST_HTTP_MAX_BODY_SIZE) {
            cwist_http_request_destroy(req);
            return NULL;
        }
 
        char *body = cwist_alloc(req->content_length + 1);
        if (!body) {
            cwist_http_request_destroy(req);
            return NULL;
        }
 
        size_t to_copy = body_received < req->content_length ? body_received : req->content_length;
        memcpy(body, header_end + 4, to_copy);
        size_t current_body_len = to_copy;
 
        /* No total cap (slow 1 GiB uploads must keep working); abort only
         * when no bytes arrive for a cumulative idle span.  Any successful
         * read resets the idle clock. */
        uint64_t body_idle_start = cwist_https_now_ms();
 
        while (current_body_len < req->content_length) {
            if (SSL_pending(conn->ssl) == 0) {
                uint64_t now = cwist_https_now_ms();
                if (now - body_idle_start >= CWIST_HTTP_BODY_IDLE_TIMEOUT_MS) {
                    cwist_free(body);
                    cwist_http_request_destroy(req);
                    return NULL;
                }
                int wait_ms = CWIST_HTTP_TIMEOUT_MS;
                uint64_t idle_left = CWIST_HTTP_BODY_IDLE_TIMEOUT_MS - (now - body_idle_start);
                if (idle_left < (uint64_t)wait_ms) wait_ms = (int)idle_left;
 
                struct pollfd pfd = {.fd = conn->fd, .events = POLLIN};
                int pret = poll(&pfd, 1, wait_ms);
                if (pret <= 0) {
                    cwist_free(body);
                    cwist_http_request_destroy(req);
                    return NULL;
                }
            }
 
            int bytes = SSL_read(conn->ssl, body + current_body_len,
                                 (int)(req->content_length - current_body_len));
            if (bytes <= 0) {
                int ssl_err = SSL_get_error(conn->ssl, bytes);
                if (ssl_err == SSL_ERROR_WANT_READ || ssl_err == SSL_ERROR_WANT_WRITE) {
                    if (cwist_ssl_wait(conn->fd, ssl_err, CWIST_HTTP_TIMEOUT_MS) != 0) {
                        cwist_free(body);
                        cwist_http_request_destroy(req);
                        return NULL;
                    }
                    continue;
                }
                cwist_free(body);
                cwist_http_request_destroy(req);
                return NULL;
            }
            current_body_len += (size_t)bytes;
            body_idle_start = cwist_https_now_ms();
        }
        body[req->content_length] = '\0';
        /* Adopt the filled buffer: one allocation, zero copies. */
        cwist_sstring_adopt_len(req->body, body, req->content_length);
 
        if (body_received > req->content_length) {
            size_t leftover_len = body_received - req->content_length;
            memmove(conn->read_buf, header_end + 4 + req->content_length, leftover_len);
            conn->buf_len = leftover_len;
        } else {
            conn->buf_len = 0;
        }
    } else {
        if (body_received > 0) {
            memmove(conn->read_buf, header_end + 4, body_received);
            conn->buf_len = body_received;
        } else {
            conn->buf_len = 0;
        }
    }
    conn->read_buf[conn->buf_len] = '\0';
 
    return req;
}
 
static int cwist_ssl_write_all(cwist_https_connection *conn, const char *buf, size_t len) {
    size_t off = 0;
    while (off < len) {
        /* SSL_write takes an int; cap each call well below INT_MAX. */
        size_t left = len - off;
        int chunk = left > 0x3fffffff ? 0x3fffffff : (int)left;
        int sent = SSL_write(conn->ssl, buf + off, chunk);
        if (sent <= 0) {
            int ssl_err = SSL_get_error(conn->ssl, sent);
            if (ssl_err == SSL_ERROR_WANT_WRITE || ssl_err == SSL_ERROR_WANT_READ) {
                if (cwist_ssl_wait(conn->fd, ssl_err, CWIST_HTTP_TIMEOUT_MS) != 0) {
                    return -1;
                }
                continue;
            }
            return -1;
        }
        off += (size_t)sent;
    }
    return 0;
}
 
cwist_error_t cwist_https_send_response(cwist_https_connection *conn, cwist_http_response *res) {
    cwist_error_t err = make_error(CWIST_ERR_INT16);
 
    if (!conn || !conn->ssl || !res) {
        err.error.err_i16 = -1;
        return err;
    }
 
    // Inject Alt-Svc when HTTP/3 is enabled so clients discover the QUIC endpoint,
    // unless the application already set or cleared the Alt-Svc header.
    if (conn->http3_enabled && !cwist_http_header_get(res->headers, "Alt-Svc")) {
        struct sockaddr_storage ss;
        socklen_t ss_len = sizeof(ss);
        int port = 443;
        if (getsockname(conn->fd, (struct sockaddr *)&ss, &ss_len) == 0) {
            if (ss.ss_family == AF_INET) {
                port = ntohs(((struct sockaddr_in *)&ss)->sin_port);
            } else if (ss.ss_family == AF_INET6) {
                port = ntohs(((struct sockaddr_in6 *)&ss)->sin6_port);
            }
        }
        char alt_svc[64];
        snprintf(alt_svc, sizeof(alt_svc), "h3=\":%d\"; ma=86400", port);
        cwist_http_header_add(&res->headers, "Alt-Svc", alt_svc);
    }
 
    // 1+2. Headers onto a stack buffer; a small body rides in the same TLS
    // record, saving one record's AEAD tag and one syscall on the common
    // short-response path. Larger bodies keep the split writes (BoringSSL
    // records a big SSL_write at 16 KiB internally).
    char header_buf[CWIST_HTTP_MAX_HEADER_SIZE];
    size_t header_len = cwist_http_serialize_headers(res, header_buf, sizeof(header_buf));
 
    const char *body_ptr = NULL;
    size_t body_len = 0;
    if (res->is_ptr_body) {
        body_ptr = (const char *)res->ptr_body;
        body_len = res->ptr_body_len;
    } else if (res->body && res->body->data) {
        body_ptr = res->body->data;
        body_len = res->body->size;
    }
 
    bool body_coalesced = false;
    if (body_ptr && body_len > 0 && body_len <= CWIST_TLS_COALESCE_MAX) {
        char *combined = (char *)cwist_alloc(header_len + body_len);
        if (combined) {
            memcpy(combined, header_buf, header_len);
            memcpy(combined + header_len, body_ptr, body_len);
            int rc = cwist_ssl_write_all(conn, combined, header_len + body_len);
            cwist_free(combined);
            if (rc != 0) {
                return make_ssl_error("SSL coalesced write failed");
            }
            body_coalesced = true;
        }
        /* Allocation failure: fall through to the split writes. */
    }
    if (!body_coalesced) {
        if (cwist_ssl_write_all(conn, header_buf, header_len) != 0) {
            return make_ssl_error("SSL header write failed");
        }
        if (body_ptr && body_len > 0) {
            if (cwist_ssl_write_all(conn, body_ptr, body_len) != 0) {
                return make_ssl_error("SSL body write failed");
            }
        }
    }
 
    // 3. File streams cannot use sendfile() through userland TLS; chunk them
    if (res->use_file_stream && res->file_stream_fd >= 0) {
        char fbuf[65536];
        size_t remaining = res->file_stream_len;
        off_t offset = res->file_stream_offset;
        while (remaining > 0) {
            size_t to_read = remaining < sizeof(fbuf) ? remaining : sizeof(fbuf);
            ssize_t n = pread(res->file_stream_fd, fbuf, to_read, offset);
            if (n < 0) {
                if (errno == EINTR) continue;
                return make_ssl_error("file stream read failed");
            }
            if (n == 0) break;
            if (cwist_ssl_write_all(conn, fbuf, (size_t)n) != 0) {
                return make_ssl_error("SSL file stream write failed");
            }
            offset += n;
            remaining -= (size_t)n;
        }
        res->file_stream_offset = offset;
    }
 
    err.error.err_i16 = 0;
    return err;
}
 
cwist_error_t cwist_https_send_response_head(cwist_https_connection *conn,
                                             cwist_http_response *res) {
    cwist_error_t err = make_error(CWIST_ERR_INT16);
 
    if (!conn || !conn->ssl || !res) {
        err.error.err_i16 = -1;
        return err;
    }
 
    if (conn->http3_enabled && !cwist_http_header_get(res->headers, "Alt-Svc")) {
        struct sockaddr_storage ss;
        socklen_t ss_len = sizeof(ss);
        int port = 443;
        if (getsockname(conn->fd, (struct sockaddr *)&ss, &ss_len) == 0) {
            if (ss.ss_family == AF_INET) {
                port = ntohs(((struct sockaddr_in *)&ss)->sin_port);
            } else if (ss.ss_family == AF_INET6) {
                port = ntohs(((struct sockaddr_in6 *)&ss)->sin6_port);
            }
        }
        char alt_svc[64];
        snprintf(alt_svc, sizeof(alt_svc), "h3=\":%d\"; ma=86400", port);
        cwist_http_header_add(&res->headers, "Alt-Svc", alt_svc);
    }
 
    char header_buf[CWIST_HTTP_MAX_HEADER_SIZE];
    size_t header_len = cwist_http_serialize_headers(res, header_buf, sizeof(header_buf));
    if (cwist_ssl_write_all(conn, header_buf, header_len) != 0) {
        return make_ssl_error("SSL header write failed");
    }
 
    err.error.err_i16 = 0;
    return err;
}
 
static void *https_thread_handler(void *arg) {
    struct https_thread_payload *payload = (struct https_thread_payload *)arg;
    cwist_https_connection *conn = NULL;
    cwist_error_t hs_err;
 
    if (payload->conn) {
        conn = payload->conn;
        hs_err = make_error(CWIST_ERR_INT16);
        hs_err.error.err_i16 = 0;
    } else if (payload->pres_ssl) {
        /* Handshake already completed by the shepherd thread. */
        hs_err = https_wrap_established(payload->ctx, payload->client_fd, payload->pres_ssl, &conn);
        if (!cwist_error_is_ok(&hs_err)) {
            SSL_free(payload->pres_ssl);
        }
    } else {
        hs_err = cwist_https_accept(payload->ctx, payload->client_fd, &conn);
    }
 
    if (cwist_error_is_ok(&hs_err)) {
        conn->pool_ctx = payload->ctx;
        conn->pool_handler = payload->handler;
        conn->pool_user_ctx = payload->user_ctx;
        t_https_parked = false;
        (void)cwist_async_https_take_deferred();
        payload->handler(conn, payload->user_ctx);
        /* A deferred conn belongs to its completion after the dispatch ack,
         * which may already have closed or resubmitted it: never read conn
         * here once it was deferred (#344). */
        bool deferred = cwist_async_https_take_deferred();
        if (t_https_parked) {
            t_https_parked = false;
        } else if (!deferred) {
            cwist_https_close_connection(conn);
        }
    } else {
        if (hs_err.errtype == CWIST_ERR_JSON) {
            cJSON_Delete(hs_err.error.err_json);
        }
        if (payload->conn) {
            cwist_https_close_connection(payload->conn);
        } else {
            close(payload->client_fd);
        }
    }
 
    cwist_free(payload);
    return NULL;
}
 
cwist_error_t cwist_https_server_loop(int server_fd, cwist_https_context *ctx,
                                      void (*handler)(cwist_https_connection *, void *),
                                      void *user_ctx) {
    cwist_error_t err = make_error(CWIST_ERR_INT16);
    if (server_fd < 0 || !ctx || !handler) {
        err.error.err_i16 = -1;
        return err;
    }
 
    if (https_pool_init() != 0) {
        err.error.err_i16 = -1;
        return err;
    }
 
    while (atomic_load(&g_cwist_running)) {
        struct sockaddr_in addr;
        socklen_t len = sizeof(addr);
        int client_fd = accept(server_fd, (struct sockaddr *)&addr, &len);
 
        if (client_fd < 0) {
            if (errno == EINTR) continue;
            if (errno == EBADF || errno == EINVAL) break;
            continue;
        }
 
        /* Reap vanished peers within ~2 minutes instead of the ~2h kernel
         * default, so dead connections cannot park pool workers forever. */
        {
            int one = 1;
            setsockopt(client_fd, SOL_SOCKET, SO_KEEPALIVE, &one, sizeof(one));
#ifdef TCP_KEEPIDLE
            int keepidle = 60;
            setsockopt(client_fd, IPPROTO_TCP, TCP_KEEPIDLE, &keepidle, sizeof(keepidle));
#endif
#ifdef TCP_KEEPINTVL
            int keepintvl = 10;
            setsockopt(client_fd, IPPROTO_TCP, TCP_KEEPINTVL, &keepintvl, sizeof(keepintvl));
#endif
#ifdef TCP_KEEPCNT
            int keepcnt = 6;
            setsockopt(client_fd, IPPROTO_TCP, TCP_KEEPCNT, &keepcnt, sizeof(keepcnt));
#endif
        }
 
        cwist_https_dispatch(client_fd, ctx, handler, user_ctx);
    }
 
    https_pool_destroy();
    return err;
}