From ac1f54b4539cf71e67a69cf41ac90b0a0c772db6 Mon Sep 17 00:00:00 2001 From: Gene Cooperman Date: Wed, 1 Jul 2026 15:32:30 -0400 Subject: [PATCH 1/6] Bypass libtsan in mc_pthread_join's timed join mc_pthread_join's RECORD loop called pthread_timedjoin_np directly, which resolves to libtsan's interceptor under a TSAN target. Its ConsumeThreadUserId trips a thread-registry CHECK (sanitizer_thread_registry.cpp:348) and aborts. Add a libpthread_timedjoin_np handle (dlsym'd from libpthread, like the mutex/cond/sem wrappers) that bypasses libtsan, and call it from mc_pthread_join's RECORD loop instead of the raw symbol. This completes end-to-end TSAN-target checkpointing under deep-debug (mcmini record mode), alongside 5be8500 (DMTCP plugin API v3->v4) and 4bf2720 (TSan-safe RECORD prologue). Verified: `mcmini -i 3 ~/dmtcp.git/test/tsan_target` runs with no SEGV/ThreadSanitizer errors, producing a valid checkpoint matching the no-mcmini baseline. Co-Authored-By: Claude Opus 4.8 (1M context) --- include/mcmini/spy/intercept/interception.h | 5 +++++ src/lib/interception.c | 7 +++++++ src/lib/wrappers.c | 5 ++++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/include/mcmini/spy/intercept/interception.h b/include/mcmini/spy/intercept/interception.h index 105f024f..0ab77d15 100644 --- a/include/mcmini/spy/intercept/interception.h +++ b/include/mcmini/spy/intercept/interception.h @@ -29,6 +29,11 @@ int libdmtcp_pthread_create(pthread_t *thread, const pthread_attr_t *attr, int pthread_join(pthread_t thread, void**); int libpthread_pthread_join(pthread_t thread, void**); int libdmtcp_pthread_join(pthread_t thread, void**); +// TSan-safe (libtsan-bypassing) handle for pthread_timedjoin_np, used by +// mc_pthread_join's RECORD loop. Calling pthread_timedjoin_np directly resolves +// to libtsan's interceptor, which trips a thread-registry CHECK. See +// TSAN-McMini-DMTCP.txt. +int libpthread_timedjoin_np(pthread_t thread, void**, const struct timespec*); int libpthread_mutex_init(pthread_mutex_t *, const pthread_mutexattr_t *); int libpthread_mutex_lock(pthread_mutex_t *); diff --git a/src/lib/interception.c b/src/lib/interception.c index 6d0b66f6..26c0cab1 100644 --- a/src/lib/interception.c +++ b/src/lib/interception.c @@ -16,6 +16,7 @@ typeof(&pthread_create) libpthread_pthread_create_ptr; typeof(&pthread_create) libdmtcp_pthread_create_ptr; typeof(&pthread_join) libpthread_pthread_join_ptr; typeof(&pthread_join) libdmtcp_pthread_join_ptr; +typeof(&pthread_timedjoin_np) libpthread_timedjoin_np_ptr; typeof(&pthread_mutex_init) pthread_mutex_init_ptr; typeof(&pthread_mutex_lock) pthread_mutex_lock_ptr; typeof(&pthread_mutex_trylock) pthread_mutex_trylock_ptr; @@ -69,6 +70,7 @@ void mc_load_intercepted_pthread_functions(void) { libpthread_pthread_create_ptr = dlsym(libpthread_handle, "pthread_create"); libpthread_pthread_join_ptr = dlsym(libpthread_handle, "pthread_join"); + libpthread_timedjoin_np_ptr = dlsym(libpthread_handle, "pthread_timedjoin_np"); pthread_mutex_init_ptr = dlsym(libpthread_handle, "pthread_mutex_init"); pthread_mutex_lock_ptr = dlsym(libpthread_handle, "pthread_mutex_lock"); pthread_mutex_trylock_ptr = dlsym(libpthread_handle, "pthread_mutex_trylock"); @@ -228,6 +230,11 @@ int libpthread_pthread_join(pthread_t thread, void **rv) { libmcmini_init(); return (*libpthread_pthread_join_ptr)(thread, rv); } +int libpthread_timedjoin_np(pthread_t thread, void **rv, + const struct timespec *abstime) { + libmcmini_init(); + return (*libpthread_timedjoin_np_ptr)(thread, rv, abstime); +} int libdmtcp_pthread_join(pthread_t thread, void **rv) { libmcmini_init(); return (*libdmtcp_pthread_join_ptr)(thread, rv); diff --git a/src/lib/wrappers.c b/src/lib/wrappers.c index a385c012..08925a3c 100644 --- a/src/lib/wrappers.c +++ b/src/lib/wrappers.c @@ -726,7 +726,10 @@ int mc_pthread_join(pthread_t t, void **rv) { struct timespec time = {.tv_sec = 2, .tv_nsec = 0}; while (1) { - int rc = pthread_timedjoin_np(t, rv, &time); + // Use the libtsan-bypassing handle: a direct pthread_timedjoin_np would + // hit libtsan's interceptor and trip its thread-registry CHECK under + // DMTCP. See TSAN-McMini-DMTCP.txt. + int rc = libpthread_timedjoin_np(t, rv, &time); if (rc == 0) { // Join succeeded libpthread_mutex_lock(&rec_list_lock); thread_record->vo.thrd_state.status = EXITED; From 2df3f55068bccc89ab1d017ff67fa39e73451c27 Mon Sep 17 00:00:00 2001 From: Gene Cooperman Date: Sun, 26 Jul 2026 11:29:17 -0400 Subject: [PATCH 2/6] Joined threads now terminate, not park forever TSAN's pthread_join() interceptor delegates to a genuine OS-level join and blocks via the kernel until the target thread actually dies -- it does not rely on its own creation-time bookkeeping. But mc_pthread_join()'s TARGET_BRANCH case only simulated success at the model level, while the joined thread was kept parked in thread_block_indefinitely() forever, so a real join on it (e.g. from TSan) could never complete. Give each thread its own exit_permission_sem (alongside its existing pthread_map entry). A finishing thread waits on it before returning; mc_pthread_join() posts it and performs a real libpthread_pthread_join() before returning, and as a bonus, pthread_join returns a return value. --- src/lib/wrappers.c | 43 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/src/lib/wrappers.c b/src/lib/wrappers.c index 08925a3c..b73b0d40 100644 --- a/src/lib/wrappers.c +++ b/src/lib/wrappers.c @@ -18,6 +18,11 @@ typedef struct pthread_map { pthread_t thread; runner_id_t value; + // Posted by whichever thread eventually calls pthread_join() on this + // one (mc_pthread_join()'s TARGET_BRANCH case), waited on by this + // thread itself before it may really terminate (mc_exit_thread_in_child()). + // See the two functions for the full rationale. + sem_t exit_permission_sem; struct pthread_map *next; } pthread_map_t; @@ -29,6 +34,7 @@ void insert_pthread_map(pthread_t t, runner_id_t v) { pthread_map_t *n = malloc(sizeof *n); n->thread = t; n->value = v; + libpthread_sem_init(&n->exit_permission_sem, 0, 0); n->next = head; head = n; pthread_rwlock_unlock(&pthread_map_lock); @@ -47,6 +53,21 @@ runner_id_t search_pthread_map(pthread_t t) { return RID_INVALID; } +// Returns the given thread's own exit_permission_sem (see pthread_map_t), +// or NULL if the thread is not (yet) registered. +sem_t *find_exit_permission_sem(pthread_t t) { + pthread_rwlock_rdlock(&pthread_map_lock); + pthread_map_t *cur = head; + while (cur) { + if (pthread_equal(cur->thread, t)) { + break; + } + cur = cur->next; + } + pthread_rwlock_unlock(&pthread_map_lock); + return cur == NULL ? NULL : &cur->exit_permission_sem; +} + MCMINI_THREAD_LOCAL runner_id_t tid_self = RID_INVALID; @@ -335,7 +356,16 @@ void mc_exit_thread_in_child(void) { thread_get_mailbox()->type = THREAD_EXIT_TYPE; thread_wake_scheduler_and_wait(); thread_awake_scheduler_for_thread_finish_transition(); - thread_block_indefinitely(); + + // Wait for whichever thread eventually calls pthread_join() on this one + // (see mc_pthread_join()'s TARGET_BRANCH case) to grant permission before + // this thread is allowed to really terminate. This keeps this thread's + // pthread_t/tid valid for exactly as long as a real, unjoined POSIX + // thread's would be -- no longer -- rather than parking it forever + // regardless of whether anyone ever joins it. + sem_t *exit_permission = find_exit_permission_sem(pthread_self()); + assert(exit_permission != NULL); + libpthread_sem_wait(exit_permission); } void mc_exit_main_thread_in_child(void) { @@ -772,7 +802,16 @@ int mc_pthread_join(pthread_t t, void **rv) { memcpy_v(thread_get_mailbox()->cnts, &rid, sizeof(runner_id_t)); thread_get_mailbox()->type = THREAD_JOIN_TYPE; thread_wake_scheduler_and_wait(); - return 0; + + // Grant the target thread permission to really terminate now that + // it's been joined (see mc_exit_thread_in_child()), then perform a + // genuine join, so its OS thread actually dies -- and *rv is + // genuinely populated -- before we return, instead of only + // simulating success at the model level. + sem_t *exit_permission = find_exit_permission_sem(t); + assert(exit_permission != NULL); + libpthread_sem_post(exit_permission); + return libpthread_pthread_join(t, rv); } default: { libc_abort(); From 0bd036baafd5eb14ca0ef98689612313e07df6b7 Mon Sep 17 00:00:00 2001 From: Gene Cooperman Date: Sun, 26 Jul 2026 11:30:32 -0400 Subject: [PATCH 3/6] Fix pthread_map_lock leak in search_pthread_map() --- src/lib/wrappers.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/wrappers.c b/src/lib/wrappers.c index b73b0d40..e2299bac 100644 --- a/src/lib/wrappers.c +++ b/src/lib/wrappers.c @@ -45,12 +45,12 @@ runner_id_t search_pthread_map(pthread_t t) { pthread_map_t *cur = head; while (cur) { if (pthread_equal(cur->thread, t)) { - return cur->value; + break; } cur = cur->next; } pthread_rwlock_unlock(&pthread_map_lock); - return RID_INVALID; + return cur == NULL ? RID_INVALID : cur->value; } // Returns the given thread's own exit_permission_sem (see pthread_map_t), From b53a77b2c4fc81b6477f4359134e274d98953d43 Mon Sep 17 00:00:00 2001 From: Gene Cooperman Date: Mon, 27 Jul 2026 00:34:27 -0400 Subject: [PATCH 4/6] DPOR backtrack replay: Report abnormal termination classic_dpor::verify_using()'s forward-exploration path catches real_world::process::termination_error and reports it via the abnormal_termination callback, letting the run end cleanly. The backtrack-replay path (coordinator::return_to_depth(), which replays prior transitions against a freshly restarted process) had no such handling, so the same exception there escaped all the way to the top-level catch-all instead. Wrap return_to_depth() the same way. found_abnormal_termination() also needed a null check: return_to_depth()'s target thread may have no pending transition in the model's current view (unlike the forward path, where the culprit is always the runner DPOR just selected as enabled). The report then falls back to a plain "no longer pending" line instead of dereferencing a null transition. --- src/mcmini/mcmini.cpp | 15 ++++++++++++--- .../model_checking/algorithms/classic_dpor.cpp | 13 ++++++++++++- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/mcmini/mcmini.cpp b/src/mcmini/mcmini.cpp index 025c28e3..26da2c67 100644 --- a/src/mcmini/mcmini.cpp +++ b/src/mcmini/mcmini.cpp @@ -142,14 +142,23 @@ void found_abnormal_termination( for (const auto& t : program_model.get_trace()) { ss << "thread " << t->get_executor() << ": " << t->to_string() << "\n"; } + // `ub.culprit` may not have a pending transition in the model's current + // view: this callback can also fire for a termination_error raised while + // `coordinator::return_to_depth()` is replaying already-recorded history + // against a freshly restarted process, and by the replay's target depth + // the culprit thread may already have exited in the model. const transition* terminator = program_model.get_pending_transition_for(ub.culprit); - ss << "thread " << terminator->get_executor() << ": " - << terminator->to_string() << "\n"; + if (terminator != nullptr) { + ss << "thread " << terminator->get_executor() << ": " + << terminator->to_string() << "\n"; + } else { + ss << "thread " << ub.culprit << ": (no longer pending)\n"; + } ss << "\nNEXT THREAD OPERATIONS\n"; for (const auto& tpair : program_model.get_pending_transitions()) { - if (tpair.first == terminator->get_executor()) { + if (terminator != nullptr && tpair.first == terminator->get_executor()) { ss << "thread " << tpair.first << ": executing" << "\n"; } else { diff --git a/src/mcmini/model_checking/algorithms/classic_dpor.cpp b/src/mcmini/model_checking/algorithms/classic_dpor.cpp index 2504f213..533ca766 100644 --- a/src/mcmini/model_checking/algorithms/classic_dpor.cpp +++ b/src/mcmini/model_checking/algorithms/classic_dpor.cpp @@ -197,7 +197,18 @@ void classic_dpor::verify_using(coordinator &coordinator, // backtracking. log_debug(dpor_logger) << "Backtracking to depth `" << (dpor_stack.size() - 1) << "`"; - coordinator.return_to_depth(dpor_stack.size() - 1); + try { + coordinator.return_to_depth(dpor_stack.size() - 1); + } catch (const real_world::process::termination_error &te) { + // The process spawned to replay history up to this depth (see + // `coordinator::return_to_depth()`) died before the replay + // finished. Report it the same way a termination during forward + // exploration is reported, rather than letting it escape + // unhandled all the way out of `verify_using()`. + if (callbacks.abnormal_termination) + callbacks.abnormal_termination(coordinator, model_checking_stats, te); + return; + } log_debug(dpor_logger) << "Finished backtracking to depth `" << (dpor_stack.size() - 1) << "`"; model_checking_stats.trace_id++; From 39d98236e998ba5b8bfb0d3c5beaa2c30c63c625 Mon Sep 17 00:00:00 2001 From: Gene Cooperman Date: Wed, 1 Jul 2026 13:47:19 -0400 Subject: [PATCH 5/6] Port libmcmini's DMTCP plugin to API v4, from v3 The TSAN-supporting DMTCP branch (tsan-phased-init) bumped the plugin API from v3 to v4, an ABI change (DmtcpPluginDescriptor_t / DmtcpUniqueProcessId, new DmtcpCkptHeader etc.). DMTCP refused to load libmcmini.so: ASSERT pluginmanager.cpp:228: incompatible DMTCP plugin API version: plugin_api=3 expected=4 Sync the vendored include/dmtcp.h to DMTCP's v4 header (correct version string and descriptor ABI), and carry forward the only McMini-specific additions -- the mcmini_virtual_pid / mcmini_real_pid macros -- updated to the v4 function names (dmtcp_{real_to_virtual,virtual_to_real}_pid became dmtcp_pid_{real_to_virtual,virtual_to_real}). Also update the two direct callers in multithreaded_fork.c. The unused dmtcp_restore_buf_* decls are dropped (not referenced by libmcmini, and gone from v4). Co-Authored-By: Claude Opus 4.8 (1M context) --- include/dmtcp.h | 102 ++++++++++++++++++++++++++++---- src/common/multithreaded_fork.c | 4 +- 2 files changed, 93 insertions(+), 13 deletions(-) diff --git a/include/dmtcp.h b/include/dmtcp.h index d0e774e4..a545c348 100644 --- a/include/dmtcp.h +++ b/include/dmtcp.h @@ -16,6 +16,8 @@ #define DMTCP_H #include +#include +#include #include #include #include @@ -43,8 +45,8 @@ # define EXTERNC #endif // ifdef __cplusplus -/* Define to the version of this package. */ -#define DMTCP_PLUGIN_API_VERSION "3" +/* Bump when DmtcpPluginDescriptor_t changes ABI. */ +#define DMTCP_PLUGIN_API_VERSION "4" #ifdef __cplusplus namespace dmtcp { @@ -145,6 +147,11 @@ typedef union _DmtcpEventData_t { struct { char *path; } realToVirtualPath, virtualToRealPath; + + struct { + pthread_t pthread; + pid_t tid; + } pthreadInfo; } DmtcpEventData_t; typedef void (*HookFunctionPtr_t)(DmtcpEvent_t, DmtcpEventData_t *); @@ -238,12 +245,78 @@ void dmtcp_initialize_plugin(void) __attribute((weak)); typedef struct DmtcpUniqueProcessId { uint64_t _hostid; // gethostid() uint64_t _time; // time() - pid_t _pid; // getpid() + + union { + pid_t _pid; // getpid() + int32_t _; + }; + uint32_t _computation_generation; // computationGeneration() } DmtcpUniqueProcessId; int dmtcp_unique_pids_equal(DmtcpUniqueProcessId a, DmtcpUniqueProcessId b); +typedef struct DmtcpInfo { + int argc; + const char **argv; +} DmtcpInfo; + +enum ElfType { + Elf_32, + Elf_64 +}; + +typedef struct { + uint64_t startAddr; + uint64_t endAddr; +} MemRegion; + +typedef void (*PostRestartFnPtr_t)(double, int); +#define DMTCP_CKPT_SIGNATURE "DMTCP_CHECKPOINT_IMAGE_v4.0\n" +typedef struct { + char ckptSignature[32]; + + DmtcpUniqueProcessId upid; + DmtcpUniqueProcessId uppid; + DmtcpUniqueProcessId compGroup; + + pid_t pid; + pid_t ppid; + pid_t sid; + pid_t gid; + pid_t fgid; + uint32_t isRootOfProcessTree; + + uint32_t numPeers; + uint32_t elfType; + + uint64_t clock_gettime_offset; + uint64_t getcpu_offset; + uint64_t gettimeofday_offset; + uint64_t time_offset; + + // Reserve 3 * 30MB for restore buffer. +#define RESTORE_BUF_TOTAL_SIZE (90 * 1024 * 1024) + MemRegion restoreBuf; + + MemRegion vdso; + MemRegion vvar; + MemRegion vvarVClock; + + uint64_t savedBrk; + uint64_t endOfStack; + + uint64_t postRestartAddr; + //void (*post_restart)(double, int); + + char procname[1024]; + char procSelfExe[1024]; + + char padding[1792]; +} DmtcpCkptHeader; + +static_assert(sizeof(DmtcpCkptHeader) == 4096, "DmtcpCkptHeader must be 4096 bytes"); + // FIXME: // If a plugin is not compiled with defined(__PIC__) and we can verify // that we're using DMTCP (environment variables), and dmtcp_is_enabled @@ -329,8 +402,6 @@ const char *dmtcp_get_ckpt_files_subdir(void); int dmtcp_should_ckpt_open_files(void); int dmtcp_allow_overwrite_with_ckpted_files(void); int dmtcp_skip_truncate_file_at_restart(const char* path); -void dmtcp_set_restore_buf_addr(void *new_addr, uint64_t len); -uint64_t dmtcp_restore_buf_len(); int dmtcp_get_ckpt_signal(void); const char *dmtcp_get_uniquepid_str(void) __attribute__((weak)); @@ -428,13 +499,22 @@ int dmtcp_protected_environ_fd(void); * discovers a pid without going through a system call (e.g., through * the proc filesystem), use this to virtualize the pid. */ -pid_t dmtcp_real_to_virtual_pid(pid_t realPid) __attribute((weak)); -pid_t dmtcp_virtual_to_real_pid(pid_t virtualPid) __attribute((weak)); - +pid_t dmtcp_pid_real_to_virtual(pid_t realPid) __attribute((weak)); +pid_t dmtcp_pid_virtual_to_real(pid_t virtualPid) __attribute((weak)); + +// Returns 1 if virtual_tid names TSAN's own background thread, else 0. +int dmtcp_is_tsan_background_thread(int virtual_tid) __attribute((weak)); +#define dmtcp_is_tsan_background_thread(virtual_tid) \ + (dmtcp_is_tsan_background_thread ? \ + dmtcp_is_tsan_background_thread(virtual_tid) : 0) + +// McMini helpers: translate pids only when running under DMTCP. +// (DMTCP plugin API v4 renamed dmtcp_{real_to_virtual,virtual_to_real}_pid to +// dmtcp_pid_{real_to_virtual,virtual_to_real}.) #define mcmini_virtual_pid(PID) \ - (dmtcp_is_enabled() ? dmtcp_real_to_virtual_pid((PID)) : (PID)) + (dmtcp_is_enabled() ? dmtcp_pid_real_to_virtual((PID)) : (PID)) #define mcmini_real_pid(PID) \ - (dmtcp_is_enabled() ? dmtcp_virtual_to_real_pid((PID)) : (PID)) + (dmtcp_is_enabled() ? dmtcp_pid_virtual_to_real((PID)) : (PID)) // bq_file -> "batch queue file"; used only by batch-queue plugin int dmtcp_is_bq_file(const char *path) __attribute((weak)); @@ -445,7 +525,7 @@ int dmtcp_bq_restore_file(const char *path, int type) __attribute((weak)); /* These next two functions are defined in contrib/ckptfile/ckptfile.cpp - * But they are currently used only in src/plugin/ipc/file/fileconnection.cpp + * But they are currently used only in src/plugin/file/fileconnection.cpp * and in a trivial fashion. These are intended for future extensions. */ int dmtcp_must_ckpt_file(const char *path) __attribute((weak)); diff --git a/src/common/multithreaded_fork.c b/src/common/multithreaded_fork.c index 7d60a103..1112c954 100644 --- a/src/common/multithreaded_fork.c +++ b/src/common/multithreaded_fork.c @@ -177,7 +177,7 @@ pid_t get_tid_from_pthread_descriptor(pthread_t pthread_descriptor) { int offset = pthreadDescriptorTidOffset(); pid_t ctid = *(pid_t*)((char*)(pthread_descriptor) + offset); #ifdef DMTCP - pid_t virttid = dmtcp_real_to_virtual_pid(ctid); + pid_t virttid = dmtcp_pid_real_to_virtual(ctid); ctid = (virttid ? virttid : ctid); #endif return ctid; @@ -209,7 +209,7 @@ int get_child_threads(int child_threads[]) { if (atoi(entry->d_name) != 0) { pid_t nexttid = atoi(entry->d_name); #ifdef DMTCP - pid_t virttid = dmtcp_real_to_virtual_pid(nexttid); + pid_t virttid = dmtcp_pid_real_to_virtual(nexttid); nexttid = (virttid ? virttid : nexttid); #endif child_threads[i++] = nexttid; From 9bd964deb239c2955676ba1b33f7b2aed694725b Mon Sep 17 00:00:00 2001 From: Gene Cooperman Date: Mon, 27 Jul 2026 01:15:46 -0400 Subject: [PATCH 6/6] Fix restart-barrier race in template_thread() template_thread() computed the number of threads to wait for by scanning /proc/self/task live, at restart time, then subtracting a blanket 2 (for itself and the checkpoint thread). DMTCP recreates checkpointed threads asynchronously via clone(), so a scan that runs before it has finished recreating all of them undercounts -- this barrier then declares "consistent state" and lets the template thread proceed before every thread has actually restarted. Confirmed via added diagnostic logging: one thread's own restart-completion signal could arrive after the barrier already released. Fix: count ALIVE THREAD entries in head_record_mode instead. That list only ever gets entries for genuine target threads (the template thread and checkpoint thread never go through libmcmini's wrapped pthread_create(), so neither is ever recorded there), and since a DMTCP checkpoint is a full memory snapshot, it's preserved exactly as-is across every restart -- immune to any restart-time scheduling race. Also remove fast_multithreaded_fork()'s #if 1/#else wrapper: the #else side held an early, never-compiled clone()-based fork prototype that predates this file's current _Fork()-based approach and was always dead code. Also remove the needless signal-mask check in threaded fork: getcontext()/setcontext() already restore a thread's blocked-signal set via uc_sigmask, even across the raw clone() used to recreate a checkpointed thread, so the thread_sigmask field and abort were unnecessary. That check ran on every restarted thread regardless of mode, so it also blocked plain (non-multithreaded-fork) restart of any target with a blocked-signal thread. --- src/lib/dmtcp-callback.c | 86 ++++++++++++---------------------------- 1 file changed, 26 insertions(+), 60 deletions(-) diff --git a/src/lib/dmtcp-callback.c b/src/lib/dmtcp-callback.c index 7438c57a..db6cb0e3 100644 --- a/src/lib/dmtcp-callback.c +++ b/src/lib/dmtcp-callback.c @@ -1,6 +1,5 @@ #define _GNU_SOURCE #include -#include #include #include // man 2 open #include @@ -42,8 +41,6 @@ struct threadinfo { // tlsAddr only used in __aarch64__ and __riscv // In fact, __riscv has the address in a normal register, restored w/ context. unsigned long int tlsAddr; - // The kernel has a process-wide sigmask, and also a per-thread sigmask. - sigset_t thread_sigmask; // glibc:pthread_create and pthread_self use this, but not the clone call: pthread_t pthread_descriptor; } threadInfos[1000]; @@ -125,23 +122,19 @@ static void saveThreadStateBeforeFork(struct threadinfo* threadInfo) { threadInfo->pthread_descriptor = pthread_self(); getTLSPointer(threadInfo); - // FIXME: Add func fo get/set signals in child thread of child process. - // and restore thread sigmask sfter setcontext. - pthread_sigmask(SIG_BLOCK, NULL, &threadInfo->thread_sigmask); - sigset_t sigtest; - pthread_sigmask(SIG_BLOCK, NULL, &sigtest); - sigdelset(&sigtest, SIG_MULTITHREADED_FORK); - if (! sigisemptyset(&sigtest)) { - fprintf(stderr, "PID %d: multithreaded_fork() not yet implemented" - " for non-empty thread signaks\n", getpid()); - libc_abort(); - } + // No manual signal-mask save/restore needed: getcontext()/setcontext() + // already save/restore the blocked-signal set via ucontext_t's uc_sigmask, + // even when setcontext() resumes on a brand-new clone()'d OS thread (see + // child_setcontext_fast() below). } static int child_setcontext_fast(void *arg) { struct threadinfo* threadInfo = arg; setTLSPointer(threadInfo); patchThreadDescriptor(threadInfo->pthread_descriptor); + // Does not return: jumps to the getcontext() call site in + // thread_handle_after_dmtcp_restart(), restoring uc_sigmask (see + // saveThreadStateBeforeFork() above) along with the rest of the context. setcontext(&(threadInfo->context)); return 0; // not reached } @@ -191,39 +184,8 @@ pid_t fast_multithreaded_fork(void) { * ret = INLINE_SYSCALL_CALL (clone, flags, 0, NULL, ctid, 0); *} *********************************************************************/ -#if 1 pid_t _Fork(); int childpid = _Fork(); -#else - // NOT YET FULLY DEVELOPED: - int flags = CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID | SIGCHLD; - int childpid; -// syscall(SYS_clone, ...); -// stack must be NULL -// https://stackoverflow.com/questions/2898579/clone-equivalent-of-fork -// But that says to use only SIGCHLD for flags, and glibc uses the above. -// But it's okay, since we're setting ctid and tls to NULL. -// FIXME: If we're going to set the last 3 args to NULL, who cares in what order they're found! -# ifdef __x86_64__ - long clone(unsigned long flags, void *stack, - int *parent_tid, int *child_tid, - unsigned long tls); -# elif defined(__aarch64__) - long clone(unsigned long flags, void *stack, - int *parent_tid, unsigned long tls, - int *child_tid); -# elif defined(__riscv) -# error Unimplemented CPU architecture -https://github.com/bminor/glibc/blob/master/sysdeps/unix/sysv/linux/riscv/clone.S -int clone(int (*fn)(void *arg), void *child_stack, int flags, void *arg, - void *parent_tidptr, void *tls, void *child_tidptr) */ - /* The syscall expects the args to be in different slots. */ - mv a0,a2 - mv a2,a4 - mv a3,a5 - mv a4,a6 -# endif -#endif if (childpid == 0) { // child process restart_child_threads_fast(); } @@ -321,23 +283,27 @@ static void *template_thread(void *unused) { // to ensure a stable recorded state is pointless: we're not going to read it // anyway! This is an only a potential optimization for later though. + // Counting via a live /proc/self/task scan is a TOCTOU race: DMTCP + // recreates checkpointed threads asynchronously via clone(), so a scan + // that runs before it has finished recreating all of them undercounts, + // and this barrier then releases before every thread has actually + // restarted (confirmed empirically: one thread's own restart-completion + // signal can arrive after this barrier already declared a "consistent + // state"). + // + // head_record_mode instead gives an exact, race-free count: it only ever + // gets THREAD entries for genuine target threads (the template thread and + // the checkpoint thread never go through libmcmini's wrapped + // pthread_create(), so neither is ever recorded here), and -- since a + // DMTCP checkpoint is a full memory snapshot -- this list is preserved + // exactly as it was at record time across every restart, with no + // dependence on restart-time scheduling. int thread_count = 0; - struct dirent *entry; - DIR *dp = opendir("/proc/self/task"); - if (dp == NULL) { - perror("opendir"); - mc_exit(EXIT_FAILURE); - } - - while ((entry = readdir(dp))) - if (strcmp(entry->d_name, ".") != 0 && strcmp(entry->d_name, "..") != 0) + for (rec_list *entry = head_record_mode; entry != NULL; entry = entry->next) { + if (entry->vo.type == THREAD && entry->vo.thrd_state.status == ALIVE) { thread_count++; - - // We don't want to count the template thread nor - // the checkpoint thread, but these will appear in - // `/proc/self/tasks` - thread_count -= 2; - closedir(dp); + } + } log_debug( "There are %d threads... waiting for them to get into a consistent " "state...\n",