Thin implementation wrapper around libttak epoch-based reclamation. More...

#include <cwist/core/mem/gc.h>
#include <cwist/core/mem/alloc.h>
#include <ttak/mem/epoch.h>
#include <pthread.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <cwist/sys/platform.h>
#include <unistd.h>

Include dependency graph for gc.c:

Go to the source code of this file.

Macros
#define	CWIST_GC_PENDING_MIN_CAP 16

#define	CWIST_GC_PENDING_KEEP_CAP 1024

Typedefs
typedef struct cwist_conn_owner	cwist_conn_owner_t

Functions
bool	cwist_gc_reg_untrack (void *ptr)
	Detach `ptr` from whichever GC context tracks it, without freeing.

void	cwist_gc (cwist_gc_t *gc, bool manual_rotation)
	Lazily initialize and optionally configure manual rotation mode.

void	cwist_gc_shutdown (cwist_gc_t *gc)
	Destroy the wrapped libttak GC state when it has been initialized.

void	cwist_gc_rotate (cwist_gc_t *gc)
	Advance the current epoch and reclaim retired nodes when eligible.

void	cwist_gc_auto_rotate (cwist_gc_t *gc, bool enabled)
	Enable auto-rotation of epoch GC.

bool	cwist_gc_auto_rotated (cwist_gc_t *gc)
	Return if epoch GC is auto-rotated. @ param gc GC context to get status.

void	cwist_reg_ptr (cwist_gc_t gc, void ptr)
	Register a pointer with zero-size accounting metadata.

void	cwist_reg_ptr_sized (cwist_gc_t gc, void ptr, size_t size)
	Register a pointer and its approximate size with the epoch GC.

ttak_epoch_gc_t *	cwist_gc_raw (cwist_gc_t *gc)
	Expose the underlying libttak epoch GC structure for advanced integrations.

void *	cwist_ebr_alloc (size_t size)
	Allocate a block that will later be released through cwist_ebr_free().

void	cwist_ebr_free (void *ptr)
	Defer release of a block until every thread that might still be reading it (inside ttak_epoch_enter()/ttak_epoch_exit()) has left.

void	cwist_gc_pipeline_tick (void)
	Advance the EBR epoch and run any callbacks that are now safe.

void	cwist_release_guard_init (cwist_release_guard_t *guard)
	Reset a release guard to its unclaimed state.

bool	cwist_release_guard_acquire (cwist_release_guard_t *guard)
	Claim a release guard.

	__attribute__ ((constructor))
	Map the guard page before main() runs, so cwist_full_gc_enabled() – called on every cwist_alloc()/cwist_free() – never needs a pthread_once/lazy-init check on its hot path (that check itself regressed the C1M reactor latency gate in CI once already).

void	cwist_full_gc (bool enable)
	Claim full-GC mode permanently, exactly once, then harden the toggle.

bool	cwist_full_gc_enabled (void)
	Query whether full-GC mode was enabled by the (single) cwist_full_gc() claim.

bool	cwist_full_gc_locked (void)
	Query whether the full-GC toggle has already been claimed and locked.

void *	cwist_full_gc_guard_page (void)
	Expose the hardened full-GC guard page for introspection/tests.

void	cwist_gc_scope_track (void *ptr)
	Track a live cwist_alloc() block so full-GC can retire it later.

bool	cwist_gc_scope_untrack (void *ptr)
	Stop tracking a block without retiring it.

bool	cwist_gc_scope_disown (void *ptr)
	Alias of cwist_gc_scope_untrack(): drop tracking without retiring.

void	cwist_gc_scope_flush (void)
	Retire every pending block this thread is still tracking.

size_t	cwist_gc_scope_pending_count (void)
	Count this thread's not-yet-retired tracked blocks.

void	cwist_conn_registry_track (void *handle, cwist_conn_close_fn close_fn)
	Track a non-memory resource for the full-GC exit sweep.

bool	cwist_conn_registry_untrack (void *handle)
	Remove a handle from the registry, whichever thread owns it.

bool	cwist_conn_registry_adopt (void *handle)
	Make the calling thread the owner of a tracked handle.

void	cwist_conn_registry_flush (void)
	Close every handle the calling thread owns, right now.

void	cwist_conn_registry_sweep_all (void)
	Close every tracked handle, whichever thread owns it.

size_t	cwist_conn_registry_pending_count (void)
	Count the tracked handles the calling thread owns.

Detailed Description

Thin implementation wrapper around libttak epoch-based reclamation.

Definition in file gc.c.

Macro Definition Documentation

◆ CWIST_GC_PENDING_KEEP_CAP

#define CWIST_GC_PENDING_KEEP_CAP 1024

A flush keeps a table up to this size for reuse and frees larger ones, so one burst of allocations does not pin a big table to the thread.

Definition at line 564 of file gc.c.

◆ CWIST_GC_PENDING_MIN_CAP

#define CWIST_GC_PENDING_MIN_CAP 16

Initial table size.

Definition at line 561 of file gc.c.

Typedef Documentation

◆ cwist_conn_owner_t

typedef struct cwist_conn_owner cwist_conn_owner_t

Per-thread owner token; its address identifies the owning thread.

Function Documentation

◆ attribute()

__attribute__ ( (constructor) )

Map the guard page before main() runs, so cwist_full_gc_enabled() – called on every cwist_alloc()/cwist_free() – never needs a pthread_once/lazy-init check on its hot path (that check itself regressed the C1M reactor latency gate in CI once already).

Definition at line 422 of file gc.c.

◆ cwist_conn_registry_adopt()

bool cwist_conn_registry_adopt ( void * handle )

Make the calling thread the owner of a tracked handle.

Parameters

handle Handle previously passed to cwist_conn_registry_track().

Returns: true when handle is tracked (and now owned by the caller); false when it is not tracked or full GC is off.

Definition at line 1025 of file gc.c.

References cap, and cwist_full_gc_enabled().

◆ cwist_conn_registry_flush()

void cwist_conn_registry_flush ( void )

Close every handle the calling thread owns, right now.

Close every handle the calling thread owns right now, without waiting for thread exit. Analogous to cwist_gc_scope_flush() but for connections.

The registry stays usable afterwards; later tracks work as before.

Definition at line 1041 of file gc.c.

◆ cwist_conn_registry_pending_count()

size_t cwist_conn_registry_pending_count ( void )

Count the tracked handles the calling thread owns.

Number of tracked connections the calling thread owns. Introspection/testing helper.

Returns: Number of owned entries; 0 when nothing is tracked.

Definition at line 1060 of file gc.c.

References cap.

◆ cwist_conn_registry_sweep_all()

void cwist_conn_registry_sweep_all ( void )

Close every tracked handle, whichever thread owns it.

Sweep every thread's pending-sweep list right now. This is what the process-exit (atexit) hook calls; exposed directly for testing, or for callers that want a deterministic sweep point earlier than actual process exit (e.g. cwist_app_destroy()).

Intended for process-exit paths. Each handle is removed before it is closed, so racing sweeps (atexit vs. a thread's own exit) close it once.

Definition at line 1052 of file gc.c.

References cap.

◆ cwist_conn_registry_track()

void cwist_conn_registry_track	(	void *	handle,
		cwist_conn_close_fn	close_fn
	)

Track a non-memory resource for the full-GC exit sweep.

Register a connection handle for the full-GC exit sweep, owned by the calling thread (full-GC mode only; a no-op otherwise, so callers can call this unconditionally at connection-open time).

No-op unless full-GC mode is enabled. Best-effort: if the table cannot grow, the handle is dropped rather than failing the caller. Tracking a handle that is already tracked updates its close_fn and makes the caller its owner.

Parameters

handle	Resource handle to close later (e.g. socket, TLS session).
close_fn	Callback invoked with `handle` during a sweep or flush.

Definition at line 984 of file gc.c.

References cap, and cwist_full_gc_enabled().

◆ cwist_conn_registry_untrack()

bool cwist_conn_registry_untrack ( void * handle )

Remove a handle from the registry, whichever thread owns it.

Remove a handle from the registry before closing it yourself through the normal explicit path.

Parameters

handle Handle previously passed to cwist_conn_registry_track().

Returns: true when found and removed; false otherwise.

Definition at line 1010 of file gc.c.

References cap.

Referenced by cwist_https_close_connection().

◆ cwist_ebr_alloc()

void * cwist_ebr_alloc ( size_t size )

Allocate a block that will later be released through cwist_ebr_free().

Allocate a block meant to be released through cwist_ebr_free().

Parameters

size	Number of bytes to allocate.

Returns: Newly allocated block, or NULL on failure.

Definition at line 331 of file gc.c.

References cwist_alloc().

◆ cwist_ebr_free()

void cwist_ebr_free ( void * ptr )

Defer release of a block until every thread that might still be reading it (inside ttak_epoch_enter()/ttak_epoch_exit()) has left.

Defer release of a block shared across threads via epoch reclamation.

Parameters

ptr	Block to retire; a no-op when NULL.

Definition at line 350 of file gc.c.

◆ cwist_full_gc()

void cwist_full_gc ( bool enable )

Claim full-GC mode permanently, exactly once, then harden the toggle.

Enable or disable full-GC mode process-wide.

The first caller stores enable into the guard page, marks the page locked, and mprotect()s it read-only; every later call is a silent no-op. Fails safe (no-op) when the guard page was never mapped. Also toggles auto-rotation on the process-wide full-GC instance.

Parameters

enable true to enable full-GC mode, false to claim-and-disable.

Definition at line 483 of file gc.c.

References cwist_gc_auto_rotate().

◆ cwist_full_gc_enabled()

bool cwist_full_gc_enabled ( void )

Query whether full-GC mode was enabled by the (single) cwist_full_gc() claim.

Check whether full-GC mode is currently enabled.

Returns: Current enabled state; false when the guard page was never mapped.

Definition at line 515 of file gc.c.

Referenced by cwist_async_defer(), cwist_conn_registry_adopt(), cwist_conn_registry_track(), cwist_h2_async_queue_enqueue(), cwist_http_async_disown_session(), cwist_https_close_connection(), cwist_https_receive_request(), cwist_io_queue_create(), cwist_io_queue_run(), cwist_io_queue_submit(), and cwist_scheduler_create().

◆ cwist_full_gc_guard_page()

void * cwist_full_gc_guard_page ( void )

Expose the hardened full-GC guard page for introspection/tests.

Raw pointer to the full-GC toggle's guard page, or NULL if the mmap() backing it failed at process startup. Advanced/testing use only – e.g. to confirm the page is actually read-only after cwist_full_gc() has been called once.

Returns: Pointer to the guard page, or NULL if mmap() failed at startup.

Definition at line 533 of file gc.c.

◆ cwist_full_gc_locked()

bool cwist_full_gc_locked ( void )

Query whether the full-GC toggle has already been claimed and locked.

Check whether the full-GC toggle has been latched (i.e. cwist_full_gc() has been called once already and every further call will be ignored). Diagnostic/testing use.

Returns: true once cwist_full_gc() has run its claiming path exactly once.

Definition at line 524 of file gc.c.

◆ cwist_gc()

void cwist_gc	(	cwist_gc_t *	gc,
		bool	manual_rotation
	)

Lazily initialize and optionally configure manual rotation mode.

Initialize (if needed) and configure the GC context.

Parameters

gc	GC context owned by the caller.
manual_rotation	When true, keep epoch advancement under explicit caller control.

Definition at line 173 of file gc.c.

References cwist_gc::impl, and cwist_gc::initialized.

◆ cwist_gc_auto_rotate()

void cwist_gc_auto_rotate	(	cwist_gc_t *	gc,
		bool	enabled
	)

Enable auto-rotation of epoch GC.

Toggle whether the GC context reclaims automatically.

Parameters

gc	GC context to enable auto-rotation.

Definition at line 212 of file gc.c.

References cwist_gc::auto_rotated, and cwist_gc::initialized.

Referenced by cwist_full_gc().

◆ cwist_gc_auto_rotated()

bool cwist_gc_auto_rotated ( cwist_gc_t * gc )

Return if epoch GC is auto-rotated. @ param gc GC context to get status.

Check whether the GC context is currently set to auto-rotate.

Definition at line 222 of file gc.c.

References cwist_gc::auto_rotated.

◆ cwist_gc_pipeline_tick()

void cwist_gc_pipeline_tick ( void )

Advance the EBR epoch and run any callbacks that are now safe.

Drive the EBR pipeline: attempt to advance the epoch and run any cwist_ebr_free() callbacks that are now safe to execute.

Definition at line 358 of file gc.c.

◆ cwist_gc_raw()

ttak_epoch_gc_t * cwist_gc_raw ( cwist_gc_t * gc )

Expose the underlying libttak epoch GC structure for advanced integrations.

Obtain the raw libttak epoch GC pointer (advanced usage).

Use this to register pointers that carry a libttak block header (ttak_mem_alloc_safe() et al.) via ttak_epoch_gc_register(): such blocks keep libttak's native teardown semantics and are released by ttak_mem_free() at destroy. cwist_reg_ptr() is the right API for cwist_alloc() blocks instead.

Parameters

gc	GC wrapper owned by CWIST.

Returns: Raw libttak GC handle, or NULL when the wrapper is unavailable.

Definition at line 321 of file gc.c.

References cwist_gc::impl, and cwist_gc::initialized.

◆ cwist_gc_reg_untrack()

bool cwist_gc_reg_untrack ( void * ptr )

Detach ptr from whichever GC context tracks it, without freeing.

Detach a pointer previously handed to a GC context through cwist_reg_ptr()/cwist_reg_ptr_sized(), without freeing it.

Removes the pointer from the registry and from its owner's libttak mem_tree (when that context is still initialized) so a later cwist_gc_shutdown() of the context no longer touches the block. Called by cwist_free() so an explicit free of a registered pointer is always safe, and by cwist_reg_ptr_sized() when registration moves to another context.

Parameters

ptr	Block to detach; a no-op when NULL.

Returns: true when ptr was registered and has been detached.

Definition at line 113 of file gc.c.

References cwist_gc::impl, and cwist_gc::initialized.

◆ cwist_gc_rotate()

void cwist_gc_rotate ( cwist_gc_t * gc )

Advance the current epoch and reclaim retired nodes when eligible.

Rotate the GC epoch and reclaim retired nodes.

Parameters

gc	GC context to rotate.

Definition at line 203 of file gc.c.

References cwist_gc::impl, and cwist_gc::initialized.

◆ cwist_gc_scope_disown()

bool cwist_gc_scope_disown ( void * ptr )

Alias of cwist_gc_scope_untrack(): drop tracking without retiring.

Remove ptr from the current thread's pending-sweep list without freeing it, because ownership is being handed off to other code that will call cwist_free() (or cwist_ebr_free()) on it later, possibly from a different thread.

Parameters

ptr	Block to untrack.

Returns: true when ptr was tracked and removed; false otherwise.

Definition at line 766 of file gc.c.

References cwist_gc_scope_untrack().

Referenced by cwist_async_defer(), cwist_h2_async_queue_enqueue(), cwist_http_async_disown_session(), cwist_https_receive_request(), cwist_io_queue_create(), cwist_io_queue_submit(), and cwist_scheduler_create().

◆ cwist_gc_scope_flush()

void cwist_gc_scope_flush ( void )

Retire every pending block this thread is still tracking.

Retire every block still on the current thread's pending-sweep list right now, instead of waiting for the thread to exit.

Each tracked block is handed to cwist_ebr_free(), so reclamation is deferred until no thread can still be reading inside an epoch.

Definition at line 776 of file gc.c.

Referenced by cwist_io_queue_run().

◆ cwist_gc_scope_pending_count()

size_t cwist_gc_scope_pending_count ( void )

Count this thread's not-yet-retired tracked blocks.

Number of blocks currently on the calling thread's pending-sweep list. Introspection/testing helper.

Returns: Number of pending entries; 0 when nothing is tracked.

Definition at line 784 of file gc.c.

◆ cwist_gc_scope_track()

void cwist_gc_scope_track ( void * ptr )

Track a live cwist_alloc() block so full-GC can retire it later.

Register a cwist_alloc() block with the current thread's pending-sweep list (full-GC mode only).

Best-effort: on allocation failure the block is left untracked rather than failing the caller's allocation. Duplicate pointers are stored once.

Parameters

ptr	Block to track; a no-op when NULL.

Definition at line 707 of file gc.c.

References cap.

◆ cwist_gc_scope_untrack()

bool cwist_gc_scope_untrack ( void * ptr )

Stop tracking a block without retiring it.

Remove a block from the current thread's pending-sweep list.

Used when the caller hands ownership elsewhere (e.g. frees through a different path); the block is removed from this thread's pending set and will not be epoch-retired by a later flush.

Parameters

ptr	Block to untrack.

Returns: true when ptr was tracked and removed; false otherwise.

Definition at line 734 of file gc.c.

References cap.

Referenced by cwist_gc_scope_disown().

◆ cwist_gc_shutdown()

void cwist_gc_shutdown ( cwist_gc_t * gc )

Destroy the wrapped libttak GC state when it has been initialized.

Shut down and free internal GC state.

Blocks still registered through cwist_reg_ptr()/cwist_reg_ptr_sized() are detached from the libttak tree and released with cwist_free() first; anything registered directly with ttak via cwist_gc_raw() is left for ttak_epoch_gc_destroy() to release with ttak_mem_free().

Parameters

gc	GC context to shut down.

Definition at line 192 of file gc.c.

References cwist_gc::impl, and cwist_gc::initialized.

◆ cwist_reg_ptr()

void cwist_reg_ptr	(	cwist_gc_t *	gc,
		void *	ptr
	)

Register a pointer with zero-size accounting metadata.

Register a pointer without specifying a size (defaults to 0).

See cwist_reg_ptr_sized() for the full contract: registering hands ownership of the block to the context, which releases it with cwist_free() at cwist_gc_shutdown().

Parameters

gc	GC context that tracks the pointer.
ptr	Pointer to retire through the epoch GC.

Definition at line 236 of file gc.c.

References cwist_reg_ptr_sized().

◆ cwist_reg_ptr_sized()

void cwist_reg_ptr_sized	(	cwist_gc_t *	gc,
		void *	ptr,
		size_t	size
	)

Register a pointer and its approximate size with the epoch GC.

Register a pointer and its size for better accounting.

The pointer must be releasable with cwist_free() (i.e. come from CWIST's allocator: cwist_alloc() and family). The wrapper records the pointer in its own registry so that cwist_gc_shutdown() can detach it from libttak's mem_tree and release it with cwist_free(); libttak's destroy-time pass frees with ttak_mem_free(), which aborts on headerless cwist_alloc blocks. Pointers allocated directly from libttak (ttak_mem_alloc_safe()) must NOT go through here; register them with the raw context instead — ttak_epoch_gc_register(cwist_gc_raw(gc), ptr, size) — so libttak's own teardown releases them.

Ownership: after registration the context owns the block and will free it at shutdown. An explicit cwist_free() first detaches it automatically; registering the same pointer with another context moves tracking there.

Parameters

gc	GC context that tracks the pointer.
ptr	Pointer to retire through the epoch GC.
size	Optional size hint associated with `ptr`.

Definition at line 261 of file gc.c.

References cwist_gc::impl, and cwist_gc::initialized.

Referenced by cwist_reg_ptr().

◆ cwist_release_guard_acquire()

bool cwist_release_guard_acquire ( cwist_release_guard_t * guard )

Claim a release guard.

Attempt to claim the guard.

Parameters

guard Guard shared by every racing caller.

Returns: true for exactly one caller across all racers; false otherwise.

Definition at line 376 of file gc.c.

◆ cwist_release_guard_init()

void cwist_release_guard_init ( cwist_release_guard_t * guard )

Reset a release guard to its unclaimed state.

Initialize a release guard to its unclaimed state.

Parameters

guard Guard to initialize.

Definition at line 366 of file gc.c.

Macros

Typedefs

Functions

Detailed Description

Macro Definition Documentation

◆ CWIST_GC_PENDING_KEEP_CAP

◆ CWIST_GC_PENDING_MIN_CAP

Typedef Documentation

◆ cwist_conn_owner_t

Function Documentation

◆ __attribute__()

◆ cwist_conn_registry_adopt()

◆ cwist_conn_registry_flush()

◆ cwist_conn_registry_pending_count()

◆ cwist_conn_registry_sweep_all()

◆ cwist_conn_registry_track()

◆ cwist_conn_registry_untrack()

◆ cwist_ebr_alloc()

◆ cwist_ebr_free()

◆ cwist_full_gc()

◆ cwist_full_gc_enabled()

◆ cwist_full_gc_guard_page()

◆ cwist_full_gc_locked()

◆ cwist_gc()

◆ cwist_gc_auto_rotate()

◆ cwist_gc_auto_rotated()

◆ cwist_gc_pipeline_tick()

◆ cwist_gc_raw()

◆ cwist_gc_reg_untrack()

◆ cwist_gc_rotate()

◆ cwist_gc_scope_disown()

◆ cwist_gc_scope_flush()

◆ cwist_gc_scope_pending_count()

◆ cwist_gc_scope_track()

◆ cwist_gc_scope_untrack()

◆ cwist_gc_shutdown()

◆ cwist_reg_ptr()

◆ cwist_reg_ptr_sized()

◆ cwist_release_guard_acquire()

◆ cwist_release_guard_init()

◆ attribute()