head 1.6; access; symbols netbsd-11-0-RC4:1.4 netbsd-11-0-RC3:1.4 netbsd-11-0-RC2:1.4 netbsd-11-0-RC1:1.4 gcc-14-3-0:1.1.1.3 perseant-exfatfs-base-20250801:1.4 netbsd-11:1.4.0.4 netbsd-11-base:1.4 gcc-12-5-0:1.1.1.2 perseant-exfatfs-base-20240630:1.4 gcc-12-4-0:1.1.1.2 perseant-exfatfs:1.4.0.2 perseant-exfatfs-base:1.4 gcc-12-3-0:1.1.1.2 gcc-10-5-0:1.1.1.1 gcc-10-4-0:1.1.1.1 cjep_sun2x:1.2.0.4 cjep_sun2x-base:1.2 cjep_staticlib_x-base1:1.2 cjep_staticlib_x:1.2.0.2 cjep_staticlib_x-base:1.2 gcc-10-3-0:1.1.1.1 FSF:1.1.1; locks; strict; comment @// @; 1.6 date 2025.10.14.20.26.37; author christos; state Exp; branches; next 1.5; commitid wFoabCCaNsagRzeG; 1.5 date 2025.09.14.00.08.58; author mrg; state Exp; branches; next 1.4; commitid x9D5QEnvbeMI4CaG; 1.4 date 2024.06.17.23.35.52; author riastradh; state Exp; branches; next 1.3; commitid Jj7AB8tFWF0OsoeF; 1.3 date 2023.07.31.01.44.57; author mrg; state Exp; branches; next 1.2; commitid q79F5Opf0FLsyTyE; 1.2 date 2021.04.11.23.54.28; author mrg; state dead; branches; next 1.1; commitid wJn7ggfUTEMOWVOC; 1.1 date 2021.04.10.22.09.22; author mrg; state Exp; branches 1.1.1.1; next ; commitid eC4g0MRpqTvEkNOC; 1.1.1.1 date 2021.04.10.22.09.22; author mrg; state Exp; branches; next 1.1.1.2; commitid eC4g0MRpqTvEkNOC; 1.1.1.2 date 2023.07.30.05.20.41; author mrg; state Exp; branches; next 1.1.1.3; commitid tk6nV4mbc9nVEMyE; 1.1.1.3 date 2025.09.13.23.45.04; author mrg; state Exp; branches; next ; commitid KwhwN4krNWa6XBaG; desc @@ 1.6 log @Use atexit(3) instead of __cxa_atexit(3) on NetBSD when dso == NULL because our __cxa_atexit asserts. @ text @//===-- tsan_interceptors_posix.cpp ---------------------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file is a part of ThreadSanitizer (TSan), a race detector. // // FIXME: move as many interceptors as possible into // sanitizer_common/sanitizer_common_interceptors.inc //===----------------------------------------------------------------------===// #include "sanitizer_common/sanitizer_atomic.h" #include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_libc.h" #include "sanitizer_common/sanitizer_linux.h" #include "sanitizer_common/sanitizer_platform_limits_netbsd.h" #include "sanitizer_common/sanitizer_platform_limits_posix.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_posix.h" #include "sanitizer_common/sanitizer_stacktrace.h" #include "sanitizer_common/sanitizer_tls_get_addr.h" #include "interception/interception.h" #include "tsan_interceptors.h" #include "tsan_interface.h" #include "tsan_platform.h" #include "tsan_suppressions.h" #include "tsan_rtl.h" #include "tsan_mman.h" #include "tsan_fd.h" #include using namespace __tsan; DECLARE_REAL(void *, memcpy, void *to, const void *from, SIZE_T size) DECLARE_REAL(void *, memset, void *block, int c, SIZE_T size) #if SANITIZER_FREEBSD || SANITIZER_APPLE #define stdout __stdoutp #define stderr __stderrp #endif #if SANITIZER_NETBSD #define dirfd(dirp) (*(int *)(dirp)) #define fileno_unlocked(fp) \ (((__sanitizer_FILE *)fp)->_file == -1 \ ? -1 \ : (int)(unsigned short)(((__sanitizer_FILE *)fp)->_file)) #define stdout ((__sanitizer_FILE*)&__sF[1]) #define stderr ((__sanitizer_FILE*)&__sF[2]) #define nanosleep __nanosleep50 #define vfork __vfork14 #endif #ifdef __mips__ const int kSigCount = 129; #else const int kSigCount = 65; #endif #ifdef __mips__ struct ucontext_t { u64 opaque[768 / sizeof(u64) + 1]; }; #else struct ucontext_t { // The size is determined by looking at sizeof of real ucontext_t on linux. u64 opaque[936 / sizeof(u64) + 1]; }; #endif #if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 || \ defined(__s390x__) #define PTHREAD_ABI_BASE "GLIBC_2.3.2" #elif defined(__aarch64__) || SANITIZER_PPC64V2 #define PTHREAD_ABI_BASE "GLIBC_2.17" #elif SANITIZER_LOONGARCH64 #define PTHREAD_ABI_BASE "GLIBC_2.36" #elif SANITIZER_RISCV64 # define PTHREAD_ABI_BASE "GLIBC_2.27" #endif extern "C" int pthread_attr_init(void *attr); extern "C" int pthread_attr_destroy(void *attr); DECLARE_REAL(int, pthread_attr_getdetachstate, void *, void *) extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize); extern "C" int pthread_atfork(void (*prepare)(void), void (*parent)(void), void (*child)(void)); extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v)); extern "C" int pthread_setspecific(unsigned key, const void *v); DECLARE_REAL(int, pthread_mutexattr_gettype, void *, void *) DECLARE_REAL(int, fflush, __sanitizer_FILE *fp) DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr size) DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) extern "C" int pthread_equal(void *t1, void *t2); extern "C" void *pthread_self(); extern "C" void _exit(int status); #if !SANITIZER_NETBSD extern "C" int fileno_unlocked(void *stream); extern "C" int dirfd(void *dirp); #endif #if SANITIZER_NETBSD extern __sanitizer_FILE __sF[]; #else extern __sanitizer_FILE *stdout, *stderr; #endif #if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD const int PTHREAD_MUTEX_RECURSIVE = 1; const int PTHREAD_MUTEX_RECURSIVE_NP = 1; #else const int PTHREAD_MUTEX_RECURSIVE = 2; const int PTHREAD_MUTEX_RECURSIVE_NP = 2; #endif #if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD const int EPOLL_CTL_ADD = 1; #endif const int SIGILL = 4; const int SIGTRAP = 5; const int SIGABRT = 6; const int SIGFPE = 8; const int SIGSEGV = 11; const int SIGPIPE = 13; const int SIGTERM = 15; #if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_NETBSD const int SIGBUS = 10; const int SIGSYS = 12; #else const int SIGBUS = 7; const int SIGSYS = 31; #endif #if SANITIZER_HAS_SIGINFO const int SI_TIMER = -2; #endif void *const MAP_FAILED = (void*)-1; #if SANITIZER_NETBSD const int PTHREAD_BARRIER_SERIAL_THREAD = 1234567; #elif !SANITIZER_APPLE const int PTHREAD_BARRIER_SERIAL_THREAD = -1; #endif const int MAP_FIXED = 0x10; typedef long long_t; typedef __sanitizer::u16 mode_t; // From /usr/include/unistd.h # define F_ULOCK 0 /* Unlock a previously locked region. */ # define F_LOCK 1 /* Lock a region for exclusive use. */ # define F_TLOCK 2 /* Test and lock a region for exclusive use. */ # define F_TEST 3 /* Test a region for other processes locks. */ #if SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_NETBSD const int SA_SIGINFO = 0x40; const int SIG_SETMASK = 3; #elif defined(__mips__) const int SA_SIGINFO = 8; const int SIG_SETMASK = 3; #else const int SA_SIGINFO = 4; const int SIG_SETMASK = 2; #endif namespace __tsan { struct SignalDesc { bool armed; __sanitizer_siginfo siginfo; ucontext_t ctx; }; struct ThreadSignalContext { int int_signal_send; SignalDesc pending_signals[kSigCount]; // emptyset and oldset are too big for stack. __sanitizer_sigset_t emptyset; __sanitizer_sigset_t oldset; }; void EnterBlockingFunc(ThreadState *thr) { for (;;) { // The order is important to not delay a signal infinitely if it's // delivered right before we set in_blocking_func. Note: we can't call // ProcessPendingSignals when in_blocking_func is set, or we can handle // a signal synchronously when we are already handling a signal. atomic_store(&thr->in_blocking_func, 1, memory_order_relaxed); if (atomic_load(&thr->pending_signals, memory_order_relaxed) == 0) break; atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed); ProcessPendingSignals(thr); } } // The sole reason tsan wraps atexit callbacks is to establish synchronization // between callback setup and callback execution. struct AtExitCtx { void (*f)(); void *arg; uptr pc; }; // InterceptorContext holds all global data required for interceptors. // It's explicitly constructed in InitializeInterceptors with placement new // and is never destroyed. This allows usage of members with non-trivial // constructors and destructors. struct InterceptorContext { // The object is 64-byte aligned, because we want hot data to be located // in a single cache line if possible (it's accessed in every interceptor). ALIGNED(64) LibIgnore libignore; __sanitizer_sigaction sigactions[kSigCount]; #if !SANITIZER_APPLE && !SANITIZER_NETBSD unsigned finalize_key; #endif Mutex atexit_mu; Vector AtExitStack; InterceptorContext() : libignore(LINKER_INITIALIZED), atexit_mu(MutexTypeAtExit), AtExitStack() {} }; static ALIGNED(64) char interceptor_placeholder[sizeof(InterceptorContext)]; InterceptorContext *interceptor_ctx() { return reinterpret_cast(&interceptor_placeholder[0]); } LibIgnore *libignore() { return &interceptor_ctx()->libignore; } void InitializeLibIgnore() { const SuppressionContext &supp = *Suppressions(); const uptr n = supp.SuppressionCount(); for (uptr i = 0; i < n; i++) { const Suppression *s = supp.SuppressionAt(i); if (0 == internal_strcmp(s->type, kSuppressionLib)) libignore()->AddIgnoredLibrary(s->templ); } if (flags()->ignore_noninstrumented_modules) libignore()->IgnoreNoninstrumentedModules(true); libignore()->OnLibraryLoaded(0); } // The following two hooks can be used by for cooperative scheduling when // locking. #ifdef TSAN_EXTERNAL_HOOKS void OnPotentiallyBlockingRegionBegin(); void OnPotentiallyBlockingRegionEnd(); #else SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionBegin() {} SANITIZER_WEAK_CXX_DEFAULT_IMPL void OnPotentiallyBlockingRegionEnd() {} #endif } // namespace __tsan static ThreadSignalContext *SigCtx(ThreadState *thr) { // This function may be called reentrantly if it is interrupted by a signal // handler. Use CAS to handle the race. uptr ctx = atomic_load(&thr->signal_ctx, memory_order_relaxed); if (ctx == 0 && !thr->is_dead) { uptr pctx = (uptr)MmapOrDie(sizeof(ThreadSignalContext), "ThreadSignalContext"); MemoryResetRange(thr, (uptr)&SigCtx, pctx, sizeof(ThreadSignalContext)); if (atomic_compare_exchange_strong(&thr->signal_ctx, &ctx, pctx, memory_order_relaxed)) { ctx = pctx; } else { UnmapOrDie((ThreadSignalContext *)pctx, sizeof(ThreadSignalContext)); } } return (ThreadSignalContext *)ctx; } ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc) : thr_(thr) { LazyInitialize(thr); if (UNLIKELY(atomic_load(&thr->in_blocking_func, memory_order_relaxed))) { // pthread_join is marked as blocking, but it's also known to call other // intercepted functions (mmap, free). If we don't reset in_blocking_func // we can get deadlocks and memory corruptions if we deliver a synchronous // signal inside of an mmap/free interceptor. // So reset it and restore it back in the destructor. // See https://github.com/google/sanitizers/issues/1540 atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed); in_blocking_func_ = true; } if (!thr_->is_inited) return; if (!thr_->ignore_interceptors) FuncEntry(thr, pc); DPrintf("#%d: intercept %s()\n", thr_->tid, fname); ignoring_ = !thr_->in_ignored_lib && (flags()->ignore_interceptors_accesses || libignore()->IsIgnored(pc, &in_ignored_lib_)); EnableIgnores(); } ScopedInterceptor::~ScopedInterceptor() { if (!thr_->is_inited) return; DisableIgnores(); if (UNLIKELY(in_blocking_func_)) EnterBlockingFunc(thr_); if (!thr_->ignore_interceptors) { ProcessPendingSignals(thr_); FuncExit(thr_); CheckedMutex::CheckNoLocks(); } } NOINLINE void ScopedInterceptor::EnableIgnoresImpl() { ThreadIgnoreBegin(thr_, 0); if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports++; if (in_ignored_lib_) { DCHECK(!thr_->in_ignored_lib); thr_->in_ignored_lib = true; } } NOINLINE void ScopedInterceptor::DisableIgnoresImpl() { ThreadIgnoreEnd(thr_); if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports--; if (in_ignored_lib_) { DCHECK(thr_->in_ignored_lib); thr_->in_ignored_lib = false; } } #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func) #if SANITIZER_FREEBSD || SANITIZER_NETBSD # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) #else # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver) #endif #if SANITIZER_FREEBSD # define TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(func) \ INTERCEPT_FUNCTION(_pthread_##func) #else # define TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(func) #endif #if SANITIZER_NETBSD # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \ INTERCEPT_FUNCTION(__libc_##func) # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \ INTERCEPT_FUNCTION(__libc_thr_##func) #else # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) #endif #define READ_STRING_OF_LEN(thr, pc, s, len, n) \ MemoryAccessRange((thr), (pc), (uptr)(s), \ common_flags()->strict_string_checks ? (len) + 1 : (n), false) #define READ_STRING(thr, pc, s, n) \ READ_STRING_OF_LEN((thr), (pc), (s), internal_strlen(s), (n)) #define BLOCK_REAL(name) (BlockingCall(thr), REAL(name)) struct BlockingCall { explicit BlockingCall(ThreadState *thr) : thr(thr) { EnterBlockingFunc(thr); // When we are in a "blocking call", we process signals asynchronously // (right when they arrive). In this context we do not expect to be // executing any user/runtime code. The known interceptor sequence when // this is not true is: pthread_join -> munmap(stack). It's fine // to ignore munmap in this case -- we handle stack shadow separately. thr->ignore_interceptors++; } ~BlockingCall() { thr->ignore_interceptors--; atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed); } ThreadState *thr; }; TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) { SCOPED_TSAN_INTERCEPTOR(sleep, sec); unsigned res = BLOCK_REAL(sleep)(sec); AfterSleep(thr, pc); return res; } TSAN_INTERCEPTOR(int, usleep, long_t usec) { SCOPED_TSAN_INTERCEPTOR(usleep, usec); int res = BLOCK_REAL(usleep)(usec); AfterSleep(thr, pc); return res; } TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) { SCOPED_TSAN_INTERCEPTOR(nanosleep, req, rem); int res = BLOCK_REAL(nanosleep)(req, rem); AfterSleep(thr, pc); return res; } TSAN_INTERCEPTOR(int, pause, int fake) { SCOPED_TSAN_INTERCEPTOR(pause, fake); return BLOCK_REAL(pause)(fake); } // Note: we specifically call the function in such strange way // with "installed_at" because in reports it will appear between // callback frames and the frame that installed the callback. static void at_exit_callback_installed_at() { AtExitCtx *ctx; { // Ensure thread-safety. Lock l(&interceptor_ctx()->atexit_mu); // Pop AtExitCtx from the top of the stack of callback functions uptr element = interceptor_ctx()->AtExitStack.Size() - 1; ctx = interceptor_ctx()->AtExitStack[element]; interceptor_ctx()->AtExitStack.PopBack(); } ThreadState *thr = cur_thread(); Acquire(thr, ctx->pc, (uptr)ctx); FuncEntry(thr, ctx->pc); ((void(*)())ctx->f)(); FuncExit(thr); Free(ctx); } static void cxa_at_exit_callback_installed_at(void *arg) { ThreadState *thr = cur_thread(); AtExitCtx *ctx = (AtExitCtx*)arg; Acquire(thr, ctx->pc, (uptr)arg); FuncEntry(thr, ctx->pc); ((void(*)(void *arg))ctx->f)(ctx->arg); FuncExit(thr); Free(ctx); } static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), void *arg, void *dso); #if !SANITIZER_ANDROID TSAN_INTERCEPTOR(int, atexit, void (*f)()) { if (in_symbolizer()) return 0; // We want to setup the atexit callback even if we are in ignored lib // or after fork. SCOPED_INTERCEPTOR_RAW(atexit, f); return setup_at_exit_wrapper(thr, GET_CALLER_PC(), (void (*)())f, 0, 0); } #endif TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) { if (in_symbolizer()) return 0; SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso); return setup_at_exit_wrapper(thr, GET_CALLER_PC(), (void (*)())f, arg, dso); } static int setup_at_exit_wrapper(ThreadState *thr, uptr pc, void(*f)(), void *arg, void *dso) { auto *ctx = New(); ctx->f = f; ctx->arg = arg; ctx->pc = pc; Release(thr, pc, (uptr)ctx); // Memory allocation in __cxa_atexit will race with free during exit, // because we do not see synchronization around atexit callback list. ThreadIgnoreBegin(thr, pc); int res; if (!dso) { // NetBSD does not preserve the 2nd argument if dso is equal to 0 // Store ctx in a local stack-like structure // Ensure thread-safety. Lock l(&interceptor_ctx()->atexit_mu); // __cxa_atexit calls calloc. If we don't ignore interceptors, we will fail // due to atexit_mu held on exit from the calloc interceptor. ScopedIgnoreInterceptors ignore; #if !SANITIZER_NETBSD res = REAL(__cxa_atexit)((void (*)(void *a))at_exit_callback_installed_at, 0, 0); #else res = REAL(atexit)((void (*)())at_exit_callback_installed_at); #endif // Push AtExitCtx on the top of the stack of callback functions if (!res) { interceptor_ctx()->AtExitStack.PushBack(ctx); } } else { res = REAL(__cxa_atexit)(cxa_at_exit_callback_installed_at, ctx, dso); } ThreadIgnoreEnd(thr); return res; } #if !SANITIZER_APPLE && !SANITIZER_NETBSD static void on_exit_callback_installed_at(int status, void *arg) { ThreadState *thr = cur_thread(); AtExitCtx *ctx = (AtExitCtx*)arg; Acquire(thr, ctx->pc, (uptr)arg); FuncEntry(thr, ctx->pc); ((void(*)(int status, void *arg))ctx->f)(status, ctx->arg); FuncExit(thr); Free(ctx); } TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) { if (in_symbolizer()) return 0; SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg); auto *ctx = New(); ctx->f = (void(*)())f; ctx->arg = arg; ctx->pc = GET_CALLER_PC(); Release(thr, pc, (uptr)ctx); // Memory allocation in __cxa_atexit will race with free during exit, // because we do not see synchronization around atexit callback list. ThreadIgnoreBegin(thr, pc); int res = REAL(on_exit)(on_exit_callback_installed_at, ctx); ThreadIgnoreEnd(thr); return res; } #define TSAN_MAYBE_INTERCEPT_ON_EXIT TSAN_INTERCEPT(on_exit) #else #define TSAN_MAYBE_INTERCEPT_ON_EXIT #endif // Cleanup old bufs. static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) { for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { JmpBuf *buf = &thr->jmp_bufs[i]; if (buf->sp <= sp) { uptr sz = thr->jmp_bufs.Size(); internal_memcpy(buf, &thr->jmp_bufs[sz - 1], sizeof(*buf)); thr->jmp_bufs.PopBack(); i--; } } } static void SetJmp(ThreadState *thr, uptr sp) { if (!thr->is_inited) // called from libc guts during bootstrap return; // Cleanup old bufs. JmpBufGarbageCollect(thr, sp); // Remember the buf. JmpBuf *buf = thr->jmp_bufs.PushBack(); buf->sp = sp; buf->shadow_stack_pos = thr->shadow_stack_pos; ThreadSignalContext *sctx = SigCtx(thr); buf->int_signal_send = sctx ? sctx->int_signal_send : 0; buf->in_blocking_func = atomic_load(&thr->in_blocking_func, memory_order_relaxed); buf->in_signal_handler = atomic_load(&thr->in_signal_handler, memory_order_relaxed); } static void LongJmp(ThreadState *thr, uptr *env) { uptr sp = ExtractLongJmpSp(env); // Find the saved buf with matching sp. for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) { JmpBuf *buf = &thr->jmp_bufs[i]; if (buf->sp == sp) { CHECK_GE(thr->shadow_stack_pos, buf->shadow_stack_pos); // Unwind the stack. while (thr->shadow_stack_pos > buf->shadow_stack_pos) FuncExit(thr); ThreadSignalContext *sctx = SigCtx(thr); if (sctx) sctx->int_signal_send = buf->int_signal_send; atomic_store(&thr->in_blocking_func, buf->in_blocking_func, memory_order_relaxed); atomic_store(&thr->in_signal_handler, buf->in_signal_handler, memory_order_relaxed); JmpBufGarbageCollect(thr, buf->sp - 1); // do not collect buf->sp return; } } Printf("ThreadSanitizer: can't find longjmp buf\n"); CHECK(0); } // FIXME: put everything below into a common extern "C" block? extern "C" void __tsan_setjmp(uptr sp) { SetJmp(cur_thread_init(), sp); } #if SANITIZER_APPLE TSAN_INTERCEPTOR(int, setjmp, void *env); TSAN_INTERCEPTOR(int, _setjmp, void *env); TSAN_INTERCEPTOR(int, sigsetjmp, void *env); #else // SANITIZER_APPLE #if SANITIZER_NETBSD #define setjmp_symname __setjmp14 #define sigsetjmp_symname __sigsetjmp14 #else #define setjmp_symname setjmp #define sigsetjmp_symname sigsetjmp #endif DEFINE_REAL(int, setjmp_symname, void *env) DEFINE_REAL(int, _setjmp, void *env) DEFINE_REAL(int, sigsetjmp_symname, void *env) #if !SANITIZER_NETBSD DEFINE_REAL(int, __sigsetjmp, void *env) #endif // The real interceptor for setjmp is special, and implemented in pure asm. We // just need to initialize the REAL functions so that they can be used in asm. static void InitializeSetjmpInterceptors() { // We can not use TSAN_INTERCEPT to get setjmp addr, because it does &setjmp and // setjmp is not present in some versions of libc. using __interception::InterceptFunction; InterceptFunction(SANITIZER_STRINGIFY(setjmp_symname), (uptr*)&REAL(setjmp_symname), 0, 0); InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); InterceptFunction(SANITIZER_STRINGIFY(sigsetjmp_symname), (uptr*)&REAL(sigsetjmp_symname), 0, 0); #if !SANITIZER_NETBSD InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); #endif } #endif // SANITIZER_APPLE #if SANITIZER_NETBSD #define longjmp_symname __longjmp14 #define siglongjmp_symname __siglongjmp14 #else #define longjmp_symname longjmp #define siglongjmp_symname siglongjmp #endif TSAN_INTERCEPTOR(void, longjmp_symname, uptr *env, int val) { // Note: if we call REAL(longjmp) in the context of ScopedInterceptor, // bad things will happen. We will jump over ScopedInterceptor dtor and can // leave thr->in_ignored_lib set. { SCOPED_INTERCEPTOR_RAW(longjmp_symname, env, val); } LongJmp(cur_thread(), env); REAL(longjmp_symname)(env, val); } TSAN_INTERCEPTOR(void, siglongjmp_symname, uptr *env, int val) { { SCOPED_INTERCEPTOR_RAW(siglongjmp_symname, env, val); } LongJmp(cur_thread(), env); REAL(siglongjmp_symname)(env, val); } #if SANITIZER_NETBSD TSAN_INTERCEPTOR(void, _longjmp, uptr *env, int val) { { SCOPED_INTERCEPTOR_RAW(_longjmp, env, val); } LongJmp(cur_thread(), env); REAL(_longjmp)(env, val); } #endif #if !SANITIZER_APPLE TSAN_INTERCEPTOR(void*, malloc, uptr size) { if (in_symbolizer()) return InternalAlloc(size); void *p = 0; { SCOPED_INTERCEPTOR_RAW(malloc, size); p = user_alloc(thr, pc, size); } invoke_malloc_hook(p, size); return p; } // In glibc<2.25, dynamic TLS blocks are allocated by __libc_memalign. Intercept // __libc_memalign so that (1) we can detect races (2) free will not be called // on libc internally allocated blocks. TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) { SCOPED_INTERCEPTOR_RAW(__libc_memalign, align, sz); return user_memalign(thr, pc, align, sz); } TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) { if (in_symbolizer()) return InternalCalloc(size, n); void *p = 0; { SCOPED_INTERCEPTOR_RAW(calloc, size, n); p = user_calloc(thr, pc, size, n); } invoke_malloc_hook(p, n * size); return p; } TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) { if (in_symbolizer()) return InternalRealloc(p, size); if (p) invoke_free_hook(p); { SCOPED_INTERCEPTOR_RAW(realloc, p, size); p = user_realloc(thr, pc, p, size); } invoke_malloc_hook(p, size); return p; } TSAN_INTERCEPTOR(void*, reallocarray, void *p, uptr size, uptr n) { if (in_symbolizer()) return InternalReallocArray(p, size, n); if (p) invoke_free_hook(p); { SCOPED_INTERCEPTOR_RAW(reallocarray, p, size, n); p = user_reallocarray(thr, pc, p, size, n); } invoke_malloc_hook(p, size); return p; } TSAN_INTERCEPTOR(void, free, void *p) { if (p == 0) return; if (in_symbolizer()) return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(free, p); user_free(thr, pc, p); } TSAN_INTERCEPTOR(void, cfree, void *p) { if (p == 0) return; if (in_symbolizer()) return InternalFree(p); invoke_free_hook(p); SCOPED_INTERCEPTOR_RAW(cfree, p); user_free(thr, pc, p); } TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) { SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p); return user_alloc_usable_size(p); } #endif TSAN_INTERCEPTOR(char *, strcpy, char *dst, const char *src) { SCOPED_TSAN_INTERCEPTOR(strcpy, dst, src); uptr srclen = internal_strlen(src); MemoryAccessRange(thr, pc, (uptr)dst, srclen + 1, true); MemoryAccessRange(thr, pc, (uptr)src, srclen + 1, false); return REAL(strcpy)(dst, src); } TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) { SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n); uptr srclen = internal_strnlen(src, n); MemoryAccessRange(thr, pc, (uptr)dst, n, true); MemoryAccessRange(thr, pc, (uptr)src, min(srclen + 1, n), false); return REAL(strncpy)(dst, src, n); } TSAN_INTERCEPTOR(char*, strdup, const char *str) { SCOPED_TSAN_INTERCEPTOR(strdup, str); // strdup will call malloc, so no instrumentation is required here. return REAL(strdup)(str); } // Zero out addr if it points into shadow memory and was provided as a hint // only, i.e., MAP_FIXED is not set. static bool fix_mmap_addr(void **addr, long_t sz, int flags) { if (*addr) { if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) { if (flags & MAP_FIXED) { errno = errno_EINVAL; return false; } else { *addr = 0; } } } return true; } template static void *mmap_interceptor(ThreadState *thr, uptr pc, Mmap real_mmap, void *addr, SIZE_T sz, int prot, int flags, int fd, OFF64_T off) { if (!fix_mmap_addr(&addr, sz, flags)) return MAP_FAILED; void *res = real_mmap(addr, sz, prot, flags, fd, off); if (res != MAP_FAILED) { if (!IsAppMem((uptr)res) || !IsAppMem((uptr)res + sz - 1)) { Report("ThreadSanitizer: mmap at bad address: addr=%p size=%p res=%p\n", addr, (void*)sz, res); Die(); } if (fd > 0) FdAccess(thr, pc, fd); MemoryRangeImitateWriteOrResetRange(thr, pc, (uptr)res, sz); } return res; } template static int munmap_interceptor(ThreadState *thr, uptr pc, Munmap real_munmap, void *addr, SIZE_T sz) { UnmapShadow(thr, (uptr)addr, sz); int res = real_munmap(addr, sz); return res; } #if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) { SCOPED_INTERCEPTOR_RAW(memalign, align, sz); return user_memalign(thr, pc, align, sz); } #define TSAN_MAYBE_INTERCEPT_MEMALIGN TSAN_INTERCEPT(memalign) #else #define TSAN_MAYBE_INTERCEPT_MEMALIGN #endif #if !SANITIZER_APPLE TSAN_INTERCEPTOR(void*, aligned_alloc, uptr align, uptr sz) { if (in_symbolizer()) return InternalAlloc(sz, nullptr, align); SCOPED_INTERCEPTOR_RAW(aligned_alloc, align, sz); return user_aligned_alloc(thr, pc, align, sz); } TSAN_INTERCEPTOR(void*, valloc, uptr sz) { if (in_symbolizer()) return InternalAlloc(sz, nullptr, GetPageSizeCached()); SCOPED_INTERCEPTOR_RAW(valloc, sz); return user_valloc(thr, pc, sz); } #endif #if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) { if (in_symbolizer()) { uptr PageSize = GetPageSizeCached(); sz = sz ? RoundUpTo(sz, PageSize) : PageSize; return InternalAlloc(sz, nullptr, PageSize); } SCOPED_INTERCEPTOR_RAW(pvalloc, sz); return user_pvalloc(thr, pc, sz); } #define TSAN_MAYBE_INTERCEPT_PVALLOC TSAN_INTERCEPT(pvalloc) #else #define TSAN_MAYBE_INTERCEPT_PVALLOC #endif #if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) { if (in_symbolizer()) { void *p = InternalAlloc(sz, nullptr, align); if (!p) return errno_ENOMEM; *memptr = p; return 0; } SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz); return user_posix_memalign(thr, pc, memptr, align, sz); } #endif // Both __cxa_guard_acquire and pthread_once 0-initialize // the object initially. pthread_once does not have any // other ABI requirements. __cxa_guard_acquire assumes // that any non-0 value in the first byte means that // initialization is completed. Contents of the remaining // bytes are up to us. constexpr u32 kGuardInit = 0; constexpr u32 kGuardDone = 1; constexpr u32 kGuardRunning = 1 << 16; constexpr u32 kGuardWaiter = 1 << 17; static int guard_acquire(ThreadState *thr, uptr pc, atomic_uint32_t *g, bool blocking_hooks = true) { if (blocking_hooks) OnPotentiallyBlockingRegionBegin(); auto on_exit = at_scope_exit([blocking_hooks] { if (blocking_hooks) OnPotentiallyBlockingRegionEnd(); }); for (;;) { u32 cmp = atomic_load(g, memory_order_acquire); if (cmp == kGuardInit) { if (atomic_compare_exchange_strong(g, &cmp, kGuardRunning, memory_order_relaxed)) return 1; } else if (cmp == kGuardDone) { if (!thr->in_ignored_lib) Acquire(thr, pc, (uptr)g); return 0; } else { if ((cmp & kGuardWaiter) || atomic_compare_exchange_strong(g, &cmp, cmp | kGuardWaiter, memory_order_relaxed)) FutexWait(g, cmp | kGuardWaiter); } } } static void guard_release(ThreadState *thr, uptr pc, atomic_uint32_t *g, u32 v) { if (!thr->in_ignored_lib) Release(thr, pc, (uptr)g); u32 old = atomic_exchange(g, v, memory_order_release); if (old & kGuardWaiter) FutexWake(g, 1 << 30); } // __cxa_guard_acquire and friends need to be intercepted in a special way - // regular interceptors will break statically-linked libstdc++. Linux // interceptors are especially defined as weak functions (so that they don't // cause link errors when user defines them as well). So they silently // auto-disable themselves when such symbol is already present in the binary. If // we link libstdc++ statically, it will bring own __cxa_guard_acquire which // will silently replace our interceptor. That's why on Linux we simply export // these interceptors with INTERFACE_ATTRIBUTE. // On OS X, we don't support statically linking, so we just use a regular // interceptor. #if SANITIZER_APPLE #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR #else #define STDCXX_INTERCEPTOR(rettype, name, ...) \ extern "C" rettype INTERFACE_ATTRIBUTE name(__VA_ARGS__) #endif // Used in thread-safe function static initialization. STDCXX_INTERCEPTOR(int, __cxa_guard_acquire, atomic_uint32_t *g) { SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g); return guard_acquire(thr, pc, g); } STDCXX_INTERCEPTOR(void, __cxa_guard_release, atomic_uint32_t *g) { SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g); guard_release(thr, pc, g, kGuardDone); } STDCXX_INTERCEPTOR(void, __cxa_guard_abort, atomic_uint32_t *g) { SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g); guard_release(thr, pc, g, kGuardInit); } namespace __tsan { void DestroyThreadState() { ThreadState *thr = cur_thread(); Processor *proc = thr->proc(); ThreadFinish(thr); ProcUnwire(proc, thr); ProcDestroy(proc); DTLS_Destroy(); cur_thread_finalize(); } void PlatformCleanUpThreadState(ThreadState *thr) { ThreadSignalContext *sctx = (ThreadSignalContext *)atomic_load( &thr->signal_ctx, memory_order_relaxed); if (sctx) { atomic_store(&thr->signal_ctx, 0, memory_order_relaxed); UnmapOrDie(sctx, sizeof(*sctx)); } } } // namespace __tsan #if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD static void thread_finalize(void *v) { uptr iter = (uptr)v; if (iter > 1) { if (pthread_setspecific(interceptor_ctx()->finalize_key, (void*)(iter - 1))) { Printf("ThreadSanitizer: failed to set thread key\n"); Die(); } return; } DestroyThreadState(); } #endif struct ThreadParam { void* (*callback)(void *arg); void *param; Tid tid; Semaphore created; Semaphore started; }; extern "C" void *__tsan_thread_start_func(void *arg) { ThreadParam *p = (ThreadParam*)arg; void* (*callback)(void *arg) = p->callback; void *param = p->param; { ThreadState *thr = cur_thread_init(); // Thread-local state is not initialized yet. ScopedIgnoreInterceptors ignore; #if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD ThreadIgnoreBegin(thr, 0); if (pthread_setspecific(interceptor_ctx()->finalize_key, (void *)GetPthreadDestructorIterations())) { Printf("ThreadSanitizer: failed to set thread key\n"); Die(); } ThreadIgnoreEnd(thr); #endif p->created.Wait(); Processor *proc = ProcCreate(); ProcWire(proc, thr); ThreadStart(thr, p->tid, GetTid(), ThreadType::Regular); p->started.Post(); } void *res = callback(param); // Prevent the callback from being tail called, // it mixes up stack traces. volatile int foo = 42; foo++; return res; } TSAN_INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*), void * param) { SCOPED_INTERCEPTOR_RAW(pthread_create, th, attr, callback, param); MaybeSpawnBackgroundThread(); if (ctx->after_multithreaded_fork) { if (flags()->die_after_fork) { Report("ThreadSanitizer: starting new threads after multi-threaded " "fork is not supported. Dying (set die_after_fork=0 to override)\n"); Die(); } else { VPrintf(1, "ThreadSanitizer: starting new threads after multi-threaded " "fork is not supported (pid %lu). Continuing because of " "die_after_fork=0, but you are on your own\n", internal_getpid()); } } __sanitizer_pthread_attr_t myattr; if (attr == 0) { pthread_attr_init(&myattr); attr = &myattr; } int detached = 0; REAL(pthread_attr_getdetachstate)(attr, &detached); AdjustStackSize(attr); ThreadParam p; p.callback = callback; p.param = param; p.tid = kMainTid; int res = -1; { // Otherwise we see false positives in pthread stack manipulation. ScopedIgnoreInterceptors ignore; ThreadIgnoreBegin(thr, pc); res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p); ThreadIgnoreEnd(thr); } if (res == 0) { p.tid = ThreadCreate(thr, pc, *(uptr *)th, IsStateDetached(detached)); CHECK_NE(p.tid, kMainTid); // Synchronization on p.tid serves two purposes: // 1. ThreadCreate must finish before the new thread starts. // Otherwise the new thread can call pthread_detach, but the pthread_t // identifier is not yet registered in ThreadRegistry by ThreadCreate. // 2. ThreadStart must finish before this thread continues. // Otherwise, this thread can call pthread_detach and reset thr->sync // before the new thread got a chance to acquire from it in ThreadStart. p.created.Post(); p.started.Wait(); } if (attr == &myattr) pthread_attr_destroy(&myattr); return res; } TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) { SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret); Tid tid = ThreadConsumeTid(thr, pc, (uptr)th); ThreadIgnoreBegin(thr, pc); int res = BLOCK_REAL(pthread_join)(th, ret); ThreadIgnoreEnd(thr); if (res == 0) { ThreadJoin(thr, pc, tid); } return res; } DEFINE_REAL_PTHREAD_FUNCTIONS TSAN_INTERCEPTOR(int, pthread_detach, void *th) { SCOPED_INTERCEPTOR_RAW(pthread_detach, th); Tid tid = ThreadConsumeTid(thr, pc, (uptr)th); int res = REAL(pthread_detach)(th); if (res == 0) { ThreadDetach(thr, pc, tid); } return res; } TSAN_INTERCEPTOR(void, pthread_exit, void *retval) { { SCOPED_INTERCEPTOR_RAW(pthread_exit, retval); #if !SANITIZER_APPLE && !SANITIZER_ANDROID CHECK_EQ(thr, reinterpret_cast((reinterpret_cast(cur_thread_placeholder) + SANITIZER_CACHE_LINE_SIZE - 1) & ~static_cast(SANITIZER_CACHE_LINE_SIZE - 1))); #endif } REAL(pthread_exit)(retval); } #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, pthread_tryjoin_np, void *th, void **ret) { SCOPED_INTERCEPTOR_RAW(pthread_tryjoin_np, th, ret); Tid tid = ThreadConsumeTid(thr, pc, (uptr)th); ThreadIgnoreBegin(thr, pc); int res = REAL(pthread_tryjoin_np)(th, ret); ThreadIgnoreEnd(thr); if (res == 0) ThreadJoin(thr, pc, tid); else ThreadNotJoined(thr, pc, tid, (uptr)th); return res; } TSAN_INTERCEPTOR(int, pthread_timedjoin_np, void *th, void **ret, const struct timespec *abstime) { SCOPED_INTERCEPTOR_RAW(pthread_timedjoin_np, th, ret, abstime); Tid tid = ThreadConsumeTid(thr, pc, (uptr)th); ThreadIgnoreBegin(thr, pc); int res = BLOCK_REAL(pthread_timedjoin_np)(th, ret, abstime); ThreadIgnoreEnd(thr); if (res == 0) ThreadJoin(thr, pc, tid); else ThreadNotJoined(thr, pc, tid, (uptr)th); return res; } #endif // Problem: // NPTL implementation of pthread_cond has 2 versions (2.2.5 and 2.3.2). // pthread_cond_t has different size in the different versions. // If call new REAL functions for old pthread_cond_t, they will corrupt memory // after pthread_cond_t (old cond is smaller). // If we call old REAL functions for new pthread_cond_t, we will lose some // functionality (e.g. old functions do not support waiting against // CLOCK_REALTIME). // Proper handling would require to have 2 versions of interceptors as well. // But this is messy, in particular requires linker scripts when sanitizer // runtime is linked into a shared library. // Instead we assume we don't have dynamic libraries built against old // pthread (2.2.5 is dated by 2002). And provide legacy_pthread_cond flag // that allows to work with old libraries (but this mode does not support // some features, e.g. pthread_condattr_getpshared). static void *init_cond(void *c, bool force = false) { // sizeof(pthread_cond_t) >= sizeof(uptr) in both versions. // So we allocate additional memory on the side large enough to hold // any pthread_cond_t object. Always call new REAL functions, but pass // the aux object to them. // Note: the code assumes that PTHREAD_COND_INITIALIZER initializes // first word of pthread_cond_t to zero. // It's all relevant only for linux. if (!common_flags()->legacy_pthread_cond) return c; atomic_uintptr_t *p = (atomic_uintptr_t*)c; uptr cond = atomic_load(p, memory_order_acquire); if (!force && cond != 0) return (void*)cond; void *newcond = WRAP(malloc)(pthread_cond_t_sz); internal_memset(newcond, 0, pthread_cond_t_sz); if (atomic_compare_exchange_strong(p, &cond, (uptr)newcond, memory_order_acq_rel)) return newcond; WRAP(free)(newcond); return (void*)cond; } namespace { template struct CondMutexUnlockCtx { ScopedInterceptor *si; ThreadState *thr; uptr pc; void *m; void *c; const Fn &fn; int Cancel() const { return fn(); } void Unlock() const; }; template void CondMutexUnlockCtx::Unlock() const { // pthread_cond_wait interceptor has enabled async signal delivery // (see BlockingCall below). Disable async signals since we are running // tsan code. Also ScopedInterceptor and BlockingCall destructors won't run // since the thread is cancelled, so we have to manually execute them // (the thread still can run some user code due to pthread_cleanup_push). CHECK_EQ(atomic_load(&thr->in_blocking_func, memory_order_relaxed), 1); atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed); MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock); // Undo BlockingCall ctor effects. thr->ignore_interceptors--; si->~ScopedInterceptor(); } } // namespace INTERCEPTOR(int, pthread_cond_init, void *c, void *a) { void *cond = init_cond(c, true); SCOPED_TSAN_INTERCEPTOR(pthread_cond_init, cond, a); MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true); return REAL(pthread_cond_init)(cond, a); } template int cond_wait(ThreadState *thr, uptr pc, ScopedInterceptor *si, const Fn &fn, void *c, void *m) { MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); MutexUnlock(thr, pc, (uptr)m); int res = 0; // This ensures that we handle mutex lock even in case of pthread_cancel. // See test/tsan/cond_cancel.cpp. { // Enable signal delivery while the thread is blocked. BlockingCall bc(thr); CondMutexUnlockCtx arg = {si, thr, pc, m, c, fn}; res = call_pthread_cancel_with_cleanup( [](void *arg) -> int { return ((const CondMutexUnlockCtx *)arg)->Cancel(); }, [](void *arg) { ((const CondMutexUnlockCtx *)arg)->Unlock(); }, &arg); } if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock); return res; } INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, cond, m); return cond_wait( thr, pc, &si, [=]() { return REAL(pthread_cond_wait)(cond, m); }, cond, m); } INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, cond, m, abstime); return cond_wait( thr, pc, &si, [=]() { return REAL(pthread_cond_timedwait)(cond, m, abstime); }, cond, m); } #if SANITIZER_LINUX INTERCEPTOR(int, pthread_cond_clockwait, void *c, void *m, __sanitizer_clockid_t clock, void *abstime) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_clockwait, cond, m, clock, abstime); return cond_wait( thr, pc, &si, [=]() { return REAL(pthread_cond_clockwait)(cond, m, clock, abstime); }, cond, m); } #define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT TSAN_INTERCEPT(pthread_cond_clockwait) #else #define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT #endif #if SANITIZER_APPLE INTERCEPTOR(int, pthread_cond_timedwait_relative_np, void *c, void *m, void *reltime) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait_relative_np, cond, m, reltime); return cond_wait( thr, pc, &si, [=]() { return REAL(pthread_cond_timedwait_relative_np)(cond, m, reltime); }, cond, m); } #endif INTERCEPTOR(int, pthread_cond_signal, void *c) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal, cond); MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); return REAL(pthread_cond_signal)(cond); } INTERCEPTOR(int, pthread_cond_broadcast, void *c) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast, cond); MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), false); return REAL(pthread_cond_broadcast)(cond); } INTERCEPTOR(int, pthread_cond_destroy, void *c) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, cond); MemoryAccessRange(thr, pc, (uptr)c, sizeof(uptr), true); int res = REAL(pthread_cond_destroy)(cond); if (common_flags()->legacy_pthread_cond) { // Free our aux cond and zero the pointer to not leave dangling pointers. WRAP(free)(cond); atomic_store((atomic_uintptr_t*)c, 0, memory_order_relaxed); } return res; } TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init, m, a); int res = REAL(pthread_mutex_init)(m, a); if (res == 0) { u32 flagz = 0; if (a) { int type = 0; if (REAL(pthread_mutexattr_gettype)(a, &type) == 0) if (type == PTHREAD_MUTEX_RECURSIVE || type == PTHREAD_MUTEX_RECURSIVE_NP) flagz |= MutexFlagWriteReentrant; } MutexCreate(thr, pc, (uptr)m, flagz); } return res; } TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m); int res = REAL(pthread_mutex_destroy)(m); if (res == 0 || res == errno_EBUSY) { MutexDestroy(thr, pc, (uptr)m); } return res; } TSAN_INTERCEPTOR(int, pthread_mutex_lock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_lock, m); MutexPreLock(thr, pc, (uptr)m); int res = REAL(pthread_mutex_lock)(m); if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); if (res == 0 || res == errno_EOWNERDEAD) MutexPostLock(thr, pc, (uptr)m); if (res == errno_EINVAL) MutexInvalidAccess(thr, pc, (uptr)m); return res; } TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m); int res = REAL(pthread_mutex_trylock)(m); if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); if (res == 0 || res == errno_EOWNERDEAD) MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); return res; } #if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime); int res = REAL(pthread_mutex_timedlock)(m, abstime); if (res == 0) { MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); } return res; } #endif TSAN_INTERCEPTOR(int, pthread_mutex_unlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_unlock, m); MutexUnlock(thr, pc, (uptr)m); int res = REAL(pthread_mutex_unlock)(m); if (res == errno_EINVAL) MutexInvalidAccess(thr, pc, (uptr)m); return res; } #if SANITIZER_GLIBC # if !__GLIBC_PREREQ(2, 34) // glibc 2.34 applies a non-default version for the two functions. They are no // longer expected to be intercepted by programs. TSAN_INTERCEPTOR(int, __pthread_mutex_lock, void *m) { SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_lock, m); MutexPreLock(thr, pc, (uptr)m); int res = REAL(__pthread_mutex_lock)(m); if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); if (res == 0 || res == errno_EOWNERDEAD) MutexPostLock(thr, pc, (uptr)m); if (res == errno_EINVAL) MutexInvalidAccess(thr, pc, (uptr)m); return res; } TSAN_INTERCEPTOR(int, __pthread_mutex_unlock, void *m) { SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_unlock, m); MutexUnlock(thr, pc, (uptr)m); int res = REAL(__pthread_mutex_unlock)(m); if (res == errno_EINVAL) MutexInvalidAccess(thr, pc, (uptr)m); return res; } # endif #endif #if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared); int res = REAL(pthread_spin_init)(m, pshared); if (res == 0) { MutexCreate(thr, pc, (uptr)m); } return res; } TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy, m); int res = REAL(pthread_spin_destroy)(m); if (res == 0) { MutexDestroy(thr, pc, (uptr)m); } return res; } TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m); MutexPreLock(thr, pc, (uptr)m); int res = REAL(pthread_spin_lock)(m); if (res == 0) { MutexPostLock(thr, pc, (uptr)m); } return res; } TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m); int res = REAL(pthread_spin_trylock)(m); if (res == 0) { MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); } return res; } TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m); MutexUnlock(thr, pc, (uptr)m); int res = REAL(pthread_spin_unlock)(m); return res; } #endif TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a); int res = REAL(pthread_rwlock_init)(m, a); if (res == 0) { MutexCreate(thr, pc, (uptr)m); } return res; } TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy, m); int res = REAL(pthread_rwlock_destroy)(m); if (res == 0) { MutexDestroy(thr, pc, (uptr)m); } return res; } TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m); MutexPreReadLock(thr, pc, (uptr)m); int res = REAL(pthread_rwlock_rdlock)(m); if (res == 0) { MutexPostReadLock(thr, pc, (uptr)m); } return res; } TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m); int res = REAL(pthread_rwlock_tryrdlock)(m); if (res == 0) { MutexPostReadLock(thr, pc, (uptr)m, MutexFlagTryLock); } return res; } #if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime); int res = REAL(pthread_rwlock_timedrdlock)(m, abstime); if (res == 0) { MutexPostReadLock(thr, pc, (uptr)m); } return res; } #endif TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m); MutexPreLock(thr, pc, (uptr)m); int res = REAL(pthread_rwlock_wrlock)(m); if (res == 0) { MutexPostLock(thr, pc, (uptr)m); } return res; } TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m); int res = REAL(pthread_rwlock_trywrlock)(m); if (res == 0) { MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); } return res; } #if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime); int res = REAL(pthread_rwlock_timedwrlock)(m, abstime); if (res == 0) { MutexPostLock(thr, pc, (uptr)m, MutexFlagTryLock); } return res; } #endif TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m); MutexReadOrWriteUnlock(thr, pc, (uptr)m); int res = REAL(pthread_rwlock_unlock)(m); return res; } #if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) { SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count); MemoryAccess(thr, pc, (uptr)b, 1, kAccessWrite); int res = REAL(pthread_barrier_init)(b, a, count); return res; } TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) { SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b); MemoryAccess(thr, pc, (uptr)b, 1, kAccessWrite); int res = REAL(pthread_barrier_destroy)(b); return res; } TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) { SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b); Release(thr, pc, (uptr)b); MemoryAccess(thr, pc, (uptr)b, 1, kAccessRead); int res = REAL(pthread_barrier_wait)(b); MemoryAccess(thr, pc, (uptr)b, 1, kAccessRead); if (res == 0 || res == PTHREAD_BARRIER_SERIAL_THREAD) { Acquire(thr, pc, (uptr)b); } return res; } #endif TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) { SCOPED_INTERCEPTOR_RAW(pthread_once, o, f); if (o == 0 || f == 0) return errno_EINVAL; atomic_uint32_t *a; if (SANITIZER_APPLE) a = static_cast((void *)((char *)o + sizeof(long_t))); else if (SANITIZER_NETBSD) a = static_cast ((void *)((char *)o + __sanitizer::pthread_mutex_t_sz)); else a = static_cast(o); // Mac OS X appears to use pthread_once() where calling BlockingRegion hooks // result in crashes due to too little stack space. if (guard_acquire(thr, pc, a, !SANITIZER_APPLE)) { (*f)(); guard_release(thr, pc, a, kGuardDone); } return 0; } #if SANITIZER_GLIBC TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) { SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf); if (fd > 0) FdAccess(thr, pc, fd); return REAL(__fxstat)(version, fd, buf); } #define TSAN_MAYBE_INTERCEPT___FXSTAT TSAN_INTERCEPT(__fxstat) #else #define TSAN_MAYBE_INTERCEPT___FXSTAT #endif TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) { #if SANITIZER_GLIBC SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf); if (fd > 0) FdAccess(thr, pc, fd); return REAL(__fxstat)(0, fd, buf); #else SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf); if (fd > 0) FdAccess(thr, pc, fd); return REAL(fstat)(fd, buf); #endif } #if SANITIZER_GLIBC TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) { SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf); if (fd > 0) FdAccess(thr, pc, fd); return REAL(__fxstat64)(version, fd, buf); } #define TSAN_MAYBE_INTERCEPT___FXSTAT64 TSAN_INTERCEPT(__fxstat64) #else #define TSAN_MAYBE_INTERCEPT___FXSTAT64 #endif #if SANITIZER_GLIBC TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) { SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf); if (fd > 0) FdAccess(thr, pc, fd); return REAL(__fxstat64)(0, fd, buf); } #define TSAN_MAYBE_INTERCEPT_FSTAT64 TSAN_INTERCEPT(fstat64) #else #define TSAN_MAYBE_INTERCEPT_FSTAT64 #endif TSAN_INTERCEPTOR(int, open, const char *name, int oflag, ...) { va_list ap; va_start(ap, oflag); mode_t mode = va_arg(ap, int); va_end(ap); SCOPED_TSAN_INTERCEPTOR(open, name, oflag, mode); READ_STRING(thr, pc, name, 0); int fd = REAL(open)(name, oflag, mode); if (fd >= 0) FdFileCreate(thr, pc, fd); return fd; } #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, open64, const char *name, int oflag, ...) { va_list ap; va_start(ap, oflag); mode_t mode = va_arg(ap, int); va_end(ap); SCOPED_TSAN_INTERCEPTOR(open64, name, oflag, mode); READ_STRING(thr, pc, name, 0); int fd = REAL(open64)(name, oflag, mode); if (fd >= 0) FdFileCreate(thr, pc, fd); return fd; } #define TSAN_MAYBE_INTERCEPT_OPEN64 TSAN_INTERCEPT(open64) #else #define TSAN_MAYBE_INTERCEPT_OPEN64 #endif TSAN_INTERCEPTOR(int, creat, const char *name, int mode) { SCOPED_TSAN_INTERCEPTOR(creat, name, mode); READ_STRING(thr, pc, name, 0); int fd = REAL(creat)(name, mode); if (fd >= 0) FdFileCreate(thr, pc, fd); return fd; } #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, creat64, const char *name, int mode) { SCOPED_TSAN_INTERCEPTOR(creat64, name, mode); READ_STRING(thr, pc, name, 0); int fd = REAL(creat64)(name, mode); if (fd >= 0) FdFileCreate(thr, pc, fd); return fd; } #define TSAN_MAYBE_INTERCEPT_CREAT64 TSAN_INTERCEPT(creat64) #else #define TSAN_MAYBE_INTERCEPT_CREAT64 #endif TSAN_INTERCEPTOR(int, dup, int oldfd) { SCOPED_TSAN_INTERCEPTOR(dup, oldfd); int newfd = REAL(dup)(oldfd); if (oldfd >= 0 && newfd >= 0 && newfd != oldfd) FdDup(thr, pc, oldfd, newfd, true); return newfd; } TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) { SCOPED_TSAN_INTERCEPTOR(dup2, oldfd, newfd); int newfd2 = REAL(dup2)(oldfd, newfd); if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd) FdDup(thr, pc, oldfd, newfd2, false); return newfd2; } #if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) { SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags); int newfd2 = REAL(dup3)(oldfd, newfd, flags); if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd) FdDup(thr, pc, oldfd, newfd2, false); return newfd2; } #endif #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) { SCOPED_TSAN_INTERCEPTOR(eventfd, initval, flags); int fd = REAL(eventfd)(initval, flags); if (fd >= 0) FdEventCreate(thr, pc, fd); return fd; } #define TSAN_MAYBE_INTERCEPT_EVENTFD TSAN_INTERCEPT(eventfd) #else #define TSAN_MAYBE_INTERCEPT_EVENTFD #endif #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) { SCOPED_INTERCEPTOR_RAW(signalfd, fd, mask, flags); FdClose(thr, pc, fd); fd = REAL(signalfd)(fd, mask, flags); if (!MustIgnoreInterceptor(thr)) FdSignalCreate(thr, pc, fd); return fd; } #define TSAN_MAYBE_INTERCEPT_SIGNALFD TSAN_INTERCEPT(signalfd) #else #define TSAN_MAYBE_INTERCEPT_SIGNALFD #endif #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, inotify_init, int fake) { SCOPED_TSAN_INTERCEPTOR(inotify_init, fake); int fd = REAL(inotify_init)(fake); if (fd >= 0) FdInotifyCreate(thr, pc, fd); return fd; } #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT TSAN_INTERCEPT(inotify_init) #else #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT #endif #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, inotify_init1, int flags) { SCOPED_TSAN_INTERCEPTOR(inotify_init1, flags); int fd = REAL(inotify_init1)(flags); if (fd >= 0) FdInotifyCreate(thr, pc, fd); return fd; } #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 TSAN_INTERCEPT(inotify_init1) #else #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 #endif TSAN_INTERCEPTOR(int, socket, int domain, int type, int protocol) { SCOPED_TSAN_INTERCEPTOR(socket, domain, type, protocol); int fd = REAL(socket)(domain, type, protocol); if (fd >= 0) FdSocketCreate(thr, pc, fd); return fd; } TSAN_INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int *fd) { SCOPED_TSAN_INTERCEPTOR(socketpair, domain, type, protocol, fd); int res = REAL(socketpair)(domain, type, protocol, fd); if (res == 0 && fd[0] >= 0 && fd[1] >= 0) FdPipeCreate(thr, pc, fd[0], fd[1]); return res; } TSAN_INTERCEPTOR(int, connect, int fd, void *addr, unsigned addrlen) { SCOPED_TSAN_INTERCEPTOR(connect, fd, addr, addrlen); FdSocketConnecting(thr, pc, fd); int res = REAL(connect)(fd, addr, addrlen); if (res == 0 && fd >= 0) FdSocketConnect(thr, pc, fd); return res; } TSAN_INTERCEPTOR(int, bind, int fd, void *addr, unsigned addrlen) { SCOPED_TSAN_INTERCEPTOR(bind, fd, addr, addrlen); int res = REAL(bind)(fd, addr, addrlen); if (fd > 0 && res == 0) FdAccess(thr, pc, fd); return res; } TSAN_INTERCEPTOR(int, listen, int fd, int backlog) { SCOPED_TSAN_INTERCEPTOR(listen, fd, backlog); int res = REAL(listen)(fd, backlog); if (fd > 0 && res == 0) FdAccess(thr, pc, fd); return res; } TSAN_INTERCEPTOR(int, close, int fd) { SCOPED_INTERCEPTOR_RAW(close, fd); if (!in_symbolizer()) FdClose(thr, pc, fd); return REAL(close)(fd); } #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, __close, int fd) { SCOPED_INTERCEPTOR_RAW(__close, fd); FdClose(thr, pc, fd); return REAL(__close)(fd); } #define TSAN_MAYBE_INTERCEPT___CLOSE TSAN_INTERCEPT(__close) #else #define TSAN_MAYBE_INTERCEPT___CLOSE #endif // glibc guts #if SANITIZER_LINUX && !SANITIZER_ANDROID TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) { SCOPED_INTERCEPTOR_RAW(__res_iclose, state, free_addr); int fds[64]; int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds)); for (int i = 0; i < cnt; i++) FdClose(thr, pc, fds[i]); REAL(__res_iclose)(state, free_addr); } #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE TSAN_INTERCEPT(__res_iclose) #else #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE #endif TSAN_INTERCEPTOR(int, pipe, int *pipefd) { SCOPED_TSAN_INTERCEPTOR(pipe, pipefd); int res = REAL(pipe)(pipefd); if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0) FdPipeCreate(thr, pc, pipefd[0], pipefd[1]); return res; } #if !SANITIZER_APPLE TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) { SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags); int res = REAL(pipe2)(pipefd, flags); if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0) FdPipeCreate(thr, pc, pipefd[0], pipefd[1]); return res; } #endif TSAN_INTERCEPTOR(int, unlink, char *path) { SCOPED_TSAN_INTERCEPTOR(unlink, path); Release(thr, pc, File2addr(path)); int res = REAL(unlink)(path); return res; } TSAN_INTERCEPTOR(void*, tmpfile, int fake) { SCOPED_TSAN_INTERCEPTOR(tmpfile, fake); void *res = REAL(tmpfile)(fake); if (res) { int fd = fileno_unlocked(res); if (fd >= 0) FdFileCreate(thr, pc, fd); } return res; } #if SANITIZER_LINUX TSAN_INTERCEPTOR(void*, tmpfile64, int fake) { SCOPED_TSAN_INTERCEPTOR(tmpfile64, fake); void *res = REAL(tmpfile64)(fake); if (res) { int fd = fileno_unlocked(res); if (fd >= 0) FdFileCreate(thr, pc, fd); } return res; } #define TSAN_MAYBE_INTERCEPT_TMPFILE64 TSAN_INTERCEPT(tmpfile64) #else #define TSAN_MAYBE_INTERCEPT_TMPFILE64 #endif static void FlushStreams() { // Flushing all the streams here may freeze the process if a child thread is // performing file stream operations at the same time. REAL(fflush)(stdout); REAL(fflush)(stderr); } TSAN_INTERCEPTOR(void, abort, int fake) { SCOPED_TSAN_INTERCEPTOR(abort, fake); FlushStreams(); REAL(abort)(fake); } TSAN_INTERCEPTOR(int, rmdir, char *path) { SCOPED_TSAN_INTERCEPTOR(rmdir, path); Release(thr, pc, Dir2addr(path)); int res = REAL(rmdir)(path); return res; } TSAN_INTERCEPTOR(int, closedir, void *dirp) { SCOPED_INTERCEPTOR_RAW(closedir, dirp); if (dirp) { int fd = dirfd(dirp); FdClose(thr, pc, fd); } return REAL(closedir)(dirp); } #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, epoll_create, int size) { SCOPED_TSAN_INTERCEPTOR(epoll_create, size); int fd = REAL(epoll_create)(size); if (fd >= 0) FdPollCreate(thr, pc, fd); return fd; } TSAN_INTERCEPTOR(int, epoll_create1, int flags) { SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags); int fd = REAL(epoll_create1)(flags); if (fd >= 0) FdPollCreate(thr, pc, fd); return fd; } TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) { SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev); if (epfd >= 0) FdAccess(thr, pc, epfd); if (epfd >= 0 && fd >= 0) FdAccess(thr, pc, fd); if (op == EPOLL_CTL_ADD && epfd >= 0) { FdPollAdd(thr, pc, epfd, fd); FdRelease(thr, pc, epfd); } int res = REAL(epoll_ctl)(epfd, op, fd, ev); return res; } TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) { SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout); if (epfd >= 0) FdAccess(thr, pc, epfd); int res = BLOCK_REAL(epoll_wait)(epfd, ev, cnt, timeout); if (res > 0 && epfd >= 0) FdAcquire(thr, pc, epfd); return res; } TSAN_INTERCEPTOR(int, epoll_pwait, int epfd, void *ev, int cnt, int timeout, void *sigmask) { SCOPED_TSAN_INTERCEPTOR(epoll_pwait, epfd, ev, cnt, timeout, sigmask); if (epfd >= 0) FdAccess(thr, pc, epfd); int res = BLOCK_REAL(epoll_pwait)(epfd, ev, cnt, timeout, sigmask); if (res > 0 && epfd >= 0) FdAcquire(thr, pc, epfd); return res; } TSAN_INTERCEPTOR(int, epoll_pwait2, int epfd, void *ev, int cnt, void *timeout, void *sigmask) { SCOPED_INTERCEPTOR_RAW(epoll_pwait2, epfd, ev, cnt, timeout, sigmask); // This function is new and may not be present in libc and/or kernel. // Since we effectively add it to libc (as will be probed by the program // using dlsym or a weak function pointer) we need to handle the case // when it's not present in the actual libc. if (!REAL(epoll_pwait2)) { errno = errno_ENOSYS; return -1; } if (MustIgnoreInterceptor(thr)) REAL(epoll_pwait2)(epfd, ev, cnt, timeout, sigmask); if (epfd >= 0) FdAccess(thr, pc, epfd); int res = BLOCK_REAL(epoll_pwait2)(epfd, ev, cnt, timeout, sigmask); if (res > 0 && epfd >= 0) FdAcquire(thr, pc, epfd); return res; } # define TSAN_MAYBE_INTERCEPT_EPOLL \ TSAN_INTERCEPT(epoll_create); \ TSAN_INTERCEPT(epoll_create1); \ TSAN_INTERCEPT(epoll_ctl); \ TSAN_INTERCEPT(epoll_wait); \ TSAN_INTERCEPT(epoll_pwait); \ TSAN_INTERCEPT(epoll_pwait2) #else #define TSAN_MAYBE_INTERCEPT_EPOLL #endif // The following functions are intercepted merely to process pending signals. // If program blocks signal X, we must deliver the signal before the function // returns. Similarly, if program unblocks a signal (or returns from sigsuspend) // it's better to deliver the signal straight away. TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) { SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask); return REAL(sigsuspend)(mask); } TSAN_INTERCEPTOR(int, sigblock, int mask) { SCOPED_TSAN_INTERCEPTOR(sigblock, mask); return REAL(sigblock)(mask); } TSAN_INTERCEPTOR(int, sigsetmask, int mask) { SCOPED_TSAN_INTERCEPTOR(sigsetmask, mask); return REAL(sigsetmask)(mask); } TSAN_INTERCEPTOR(int, pthread_sigmask, int how, const __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset) { SCOPED_TSAN_INTERCEPTOR(pthread_sigmask, how, set, oldset); return REAL(pthread_sigmask)(how, set, oldset); } namespace __tsan { static void ReportErrnoSpoiling(ThreadState *thr, uptr pc, int sig) { VarSizeStackTrace stack; // StackTrace::GetNestInstructionPc(pc) is used because return address is // expected, OutputReport() will undo this. ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack); ThreadRegistryLock l(&ctx->thread_registry); ScopedReport rep(ReportTypeErrnoInSignal); rep.SetSigNum(sig); if (!IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack)) { rep.AddStack(stack, true); OutputReport(thr, rep); } } static void CallUserSignalHandler(ThreadState *thr, bool sync, bool acquire, int sig, __sanitizer_siginfo *info, void *uctx) { CHECK(thr->slot); __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions; if (acquire) Acquire(thr, 0, (uptr)&sigactions[sig]); // Signals are generally asynchronous, so if we receive a signals when // ignores are enabled we should disable ignores. This is critical for sync // and interceptors, because otherwise we can miss synchronization and report // false races. int ignore_reads_and_writes = thr->ignore_reads_and_writes; int ignore_interceptors = thr->ignore_interceptors; int ignore_sync = thr->ignore_sync; // For symbolizer we only process SIGSEGVs synchronously // (bug in symbolizer or in tsan). But we want to reset // in_symbolizer to fail gracefully. Symbolizer and user code // use different memory allocators, so if we don't reset // in_symbolizer we can get memory allocated with one being // feed with another, which can cause more crashes. int in_symbolizer = thr->in_symbolizer; if (!ctx->after_multithreaded_fork) { thr->ignore_reads_and_writes = 0; thr->fast_state.ClearIgnoreBit(); thr->ignore_interceptors = 0; thr->ignore_sync = 0; thr->in_symbolizer = 0; } // Ensure that the handler does not spoil errno. const int saved_errno = errno; errno = 99; // This code races with sigaction. Be careful to not read sa_sigaction twice. // Also need to remember pc for reporting before the call, // because the handler can reset it. volatile uptr pc = (sigactions[sig].sa_flags & SA_SIGINFO) ? (uptr)sigactions[sig].sigaction : (uptr)sigactions[sig].handler; if (pc != sig_dfl && pc != sig_ign) { // The callback can be either sa_handler or sa_sigaction. // They have different signatures, but we assume that passing // additional arguments to sa_handler works and is harmless. ((__sanitizer_sigactionhandler_ptr)pc)(sig, info, uctx); } if (!ctx->after_multithreaded_fork) { thr->ignore_reads_and_writes = ignore_reads_and_writes; if (ignore_reads_and_writes) thr->fast_state.SetIgnoreBit(); thr->ignore_interceptors = ignore_interceptors; thr->ignore_sync = ignore_sync; thr->in_symbolizer = in_symbolizer; } // We do not detect errno spoiling for SIGTERM, // because some SIGTERM handlers do spoil errno but reraise SIGTERM, // tsan reports false positive in such case. // It's difficult to properly detect this situation (reraise), // because in async signal processing case (when handler is called directly // from rtl_generic_sighandler) we have not yet received the reraised // signal; and it looks too fragile to intercept all ways to reraise a signal. if (ShouldReport(thr, ReportTypeErrnoInSignal) && !sync && sig != SIGTERM && errno != 99) ReportErrnoSpoiling(thr, pc, sig); errno = saved_errno; } void ProcessPendingSignalsImpl(ThreadState *thr) { atomic_store(&thr->pending_signals, 0, memory_order_relaxed); ThreadSignalContext *sctx = SigCtx(thr); if (sctx == 0) return; atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed); internal_sigfillset(&sctx->emptyset); int res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->emptyset, &sctx->oldset); CHECK_EQ(res, 0); for (int sig = 0; sig < kSigCount; sig++) { SignalDesc *signal = &sctx->pending_signals[sig]; if (signal->armed) { signal->armed = false; CallUserSignalHandler(thr, false, true, sig, &signal->siginfo, &signal->ctx); } } res = REAL(pthread_sigmask)(SIG_SETMASK, &sctx->oldset, 0); CHECK_EQ(res, 0); atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed); } } // namespace __tsan static bool is_sync_signal(ThreadSignalContext *sctx, int sig, __sanitizer_siginfo *info) { // If we are sending signal to ourselves, we must process it now. if (sctx && sig == sctx->int_signal_send) return true; #if SANITIZER_HAS_SIGINFO // POSIX timers can be configured to send any kind of signal; however, it // doesn't make any sense to consider a timer signal as synchronous! if (info->si_code == SI_TIMER) return false; #endif return sig == SIGSEGV || sig == SIGBUS || sig == SIGILL || sig == SIGTRAP || sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS; } void sighandler(int sig, __sanitizer_siginfo *info, void *ctx) { ThreadState *thr = cur_thread_init(); ThreadSignalContext *sctx = SigCtx(thr); if (sig < 0 || sig >= kSigCount) { VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig); return; } // Don't mess with synchronous signals. const bool sync = is_sync_signal(sctx, sig, info); if (sync || // If we are in blocking function, we can safely process it now // (but check if we are in a recursive interceptor, // i.e. pthread_join()->munmap()). atomic_load(&thr->in_blocking_func, memory_order_relaxed)) { atomic_fetch_add(&thr->in_signal_handler, 1, memory_order_relaxed); if (atomic_load(&thr->in_blocking_func, memory_order_relaxed)) { atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed); CallUserSignalHandler(thr, sync, true, sig, info, ctx); atomic_store(&thr->in_blocking_func, 1, memory_order_relaxed); } else { // Be very conservative with when we do acquire in this case. // It's unsafe to do acquire in async handlers, because ThreadState // can be in inconsistent state. // SIGSYS looks relatively safe -- it's synchronous and can actually // need some global state. bool acq = (sig == SIGSYS); CallUserSignalHandler(thr, sync, acq, sig, info, ctx); } atomic_fetch_add(&thr->in_signal_handler, -1, memory_order_relaxed); return; } if (sctx == 0) return; SignalDesc *signal = &sctx->pending_signals[sig]; if (signal->armed == false) { signal->armed = true; internal_memcpy(&signal->siginfo, info, sizeof(*info)); internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx)); atomic_store(&thr->pending_signals, 1, memory_order_relaxed); } } TSAN_INTERCEPTOR(int, raise, int sig) { SCOPED_TSAN_INTERCEPTOR(raise, sig); ThreadSignalContext *sctx = SigCtx(thr); CHECK_NE(sctx, 0); int prev = sctx->int_signal_send; sctx->int_signal_send = sig; int res = REAL(raise)(sig); CHECK_EQ(sctx->int_signal_send, sig); sctx->int_signal_send = prev; return res; } TSAN_INTERCEPTOR(int, kill, int pid, int sig) { SCOPED_TSAN_INTERCEPTOR(kill, pid, sig); ThreadSignalContext *sctx = SigCtx(thr); CHECK_NE(sctx, 0); int prev = sctx->int_signal_send; if (pid == (int)internal_getpid()) { sctx->int_signal_send = sig; } int res = REAL(kill)(pid, sig); if (pid == (int)internal_getpid()) { CHECK_EQ(sctx->int_signal_send, sig); sctx->int_signal_send = prev; } return res; } TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) { SCOPED_TSAN_INTERCEPTOR(pthread_kill, tid, sig); ThreadSignalContext *sctx = SigCtx(thr); CHECK_NE(sctx, 0); int prev = sctx->int_signal_send; bool self = pthread_equal(tid, pthread_self()); if (self) sctx->int_signal_send = sig; int res = REAL(pthread_kill)(tid, sig); if (self) { CHECK_EQ(sctx->int_signal_send, sig); sctx->int_signal_send = prev; } return res; } TSAN_INTERCEPTOR(int, gettimeofday, void *tv, void *tz) { SCOPED_TSAN_INTERCEPTOR(gettimeofday, tv, tz); // It's intercepted merely to process pending signals. return REAL(gettimeofday)(tv, tz); } TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service, void *hints, void *rv) { SCOPED_TSAN_INTERCEPTOR(getaddrinfo, node, service, hints, rv); // We miss atomic synchronization in getaddrinfo, // and can report false race between malloc and free // inside of getaddrinfo. So ignore memory accesses. ThreadIgnoreBegin(thr, pc); int res = REAL(getaddrinfo)(node, service, hints, rv); ThreadIgnoreEnd(thr); return res; } TSAN_INTERCEPTOR(int, fork, int fake) { if (in_symbolizer()) return REAL(fork)(fake); SCOPED_INTERCEPTOR_RAW(fork, fake); return REAL(fork)(fake); } void atfork_prepare() { if (in_symbolizer()) return; ThreadState *thr = cur_thread(); const uptr pc = StackTrace::GetCurrentPc(); ForkBefore(thr, pc); } void atfork_parent() { if (in_symbolizer()) return; ThreadState *thr = cur_thread(); const uptr pc = StackTrace::GetCurrentPc(); ForkParentAfter(thr, pc); } void atfork_child() { if (in_symbolizer()) return; ThreadState *thr = cur_thread(); const uptr pc = StackTrace::GetCurrentPc(); ForkChildAfter(thr, pc, true); FdOnFork(thr, pc); } #if !SANITIZER_IOS TSAN_INTERCEPTOR(int, vfork, int fake) { // Some programs (e.g. openjdk) call close for all file descriptors // in the child process. Under tsan it leads to false positives, because // address space is shared, so the parent process also thinks that // the descriptors are closed (while they are actually not). // This leads to false positives due to missed synchronization. // Strictly saying this is undefined behavior, because vfork child is not // allowed to call any functions other than exec/exit. But this is what // openjdk does, so we want to handle it. // We could disable interceptors in the child process. But it's not possible // to simply intercept and wrap vfork, because vfork child is not allowed // to return from the function that calls vfork, and that's exactly what // we would do. So this would require some assembly trickery as well. // Instead we simply turn vfork into fork. return WRAP(fork)(fake); } #endif #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, clone, int (*fn)(void *), void *stack, int flags, void *arg, int *parent_tid, void *tls, pid_t *child_tid) { SCOPED_INTERCEPTOR_RAW(clone, fn, stack, flags, arg, parent_tid, tls, child_tid); struct Arg { int (*fn)(void *); void *arg; }; auto wrapper = +[](void *p) -> int { auto *thr = cur_thread(); uptr pc = GET_CURRENT_PC(); // Start the background thread for fork, but not for clone. // For fork we did this always and it's known to work (or user code has // adopted). But if we do this for the new clone interceptor some code // (sandbox2) fails. So model we used to do for years and don't start the // background thread after clone. ForkChildAfter(thr, pc, false); FdOnFork(thr, pc); auto *arg = static_cast(p); return arg->fn(arg->arg); }; ForkBefore(thr, pc); Arg arg_wrapper = {fn, arg}; int pid = REAL(clone)(wrapper, stack, flags, &arg_wrapper, parent_tid, tls, child_tid); ForkParentAfter(thr, pc); return pid; } #endif #if !SANITIZER_APPLE && !SANITIZER_ANDROID typedef int (*dl_iterate_phdr_cb_t)(__sanitizer_dl_phdr_info *info, SIZE_T size, void *data); struct dl_iterate_phdr_data { ThreadState *thr; uptr pc; dl_iterate_phdr_cb_t cb; void *data; }; static bool IsAppNotRodata(uptr addr) { return IsAppMem(addr) && *MemToShadow(addr) != Shadow::kRodata; } static int dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size, void *data) { dl_iterate_phdr_data *cbdata = (dl_iterate_phdr_data *)data; // dlopen/dlclose allocate/free dynamic-linker-internal memory, which is later // accessible in dl_iterate_phdr callback. But we don't see synchronization // inside of dynamic linker, so we "unpoison" it here in order to not // produce false reports. Ignoring malloc/free in dlopen/dlclose is not enough // because some libc functions call __libc_dlopen. if (info && IsAppNotRodata((uptr)info->dlpi_name)) MemoryResetRange(cbdata->thr, cbdata->pc, (uptr)info->dlpi_name, internal_strlen(info->dlpi_name)); int res = cbdata->cb(info, size, cbdata->data); // Perform the check one more time in case info->dlpi_name was overwritten // by user callback. if (info && IsAppNotRodata((uptr)info->dlpi_name)) MemoryResetRange(cbdata->thr, cbdata->pc, (uptr)info->dlpi_name, internal_strlen(info->dlpi_name)); return res; } TSAN_INTERCEPTOR(int, dl_iterate_phdr, dl_iterate_phdr_cb_t cb, void *data) { SCOPED_TSAN_INTERCEPTOR(dl_iterate_phdr, cb, data); dl_iterate_phdr_data cbdata; cbdata.thr = thr; cbdata.pc = pc; cbdata.cb = cb; cbdata.data = data; int res = REAL(dl_iterate_phdr)(dl_iterate_phdr_cb, &cbdata); return res; } #endif static int OnExit(ThreadState *thr) { int status = Finalize(thr); FlushStreams(); return status; } #if !SANITIZER_APPLE static void HandleRecvmsg(ThreadState *thr, uptr pc, __sanitizer_msghdr *msg) { int fds[64]; int cnt = ExtractRecvmsgFDs(msg, fds, ARRAY_SIZE(fds)); for (int i = 0; i < cnt; i++) FdEventCreate(thr, pc, fds[i]); } #endif #include "sanitizer_common/sanitizer_platform_interceptors.h" // Causes interceptor recursion (getaddrinfo() and fopen()) #undef SANITIZER_INTERCEPT_GETADDRINFO // We define our own. #if SANITIZER_INTERCEPT_TLS_GET_ADDR #define NEED_TLS_GET_ADDR #endif #undef SANITIZER_INTERCEPT_TLS_GET_ADDR #define SANITIZER_INTERCEPT_TLS_GET_OFFSET 1 #undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK #define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ INTERCEPT_FUNCTION_VER(name, ver) #define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \ (INTERCEPT_FUNCTION_VER(name, ver) || INTERCEPT_FUNCTION(name)) #define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \ SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \ TsanInterceptorContext _ctx = {thr, pc}; \ ctx = (void *)&_ctx; \ (void)ctx; #define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \ if (path) \ Acquire(thr, pc, File2addr(path)); \ if (file) { \ int fd = fileno_unlocked(file); \ if (fd >= 0) FdFileCreate(thr, pc, fd); \ } #define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \ if (file) { \ int fd = fileno_unlocked(file); \ FdClose(thr, pc, fd); \ } #define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \ ({ \ CheckNoDeepBind(filename, flag); \ ThreadIgnoreBegin(thr, 0); \ void *res = REAL(dlopen)(filename, flag); \ ThreadIgnoreEnd(thr); \ res; \ }) // Ignore interceptors in OnLibraryLoaded()/Unloaded(). These hooks use code // (ListOfModules::init, MemoryMappingLayout::DumpListOfModules) that make // intercepted calls, which can cause deadlockes with ReportRace() which also // uses this code. #define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \ ({ \ ScopedIgnoreInterceptors ignore_interceptors; \ libignore()->OnLibraryLoaded(filename); \ }) #define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \ ({ \ ScopedIgnoreInterceptors ignore_interceptors; \ libignore()->OnLibraryUnloaded(); \ }) #define COMMON_INTERCEPTOR_ACQUIRE(ctx, u) \ Acquire(((TsanInterceptorContext *) ctx)->thr, pc, u) #define COMMON_INTERCEPTOR_RELEASE(ctx, u) \ Release(((TsanInterceptorContext *) ctx)->thr, pc, u) #define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \ Acquire(((TsanInterceptorContext *) ctx)->thr, pc, Dir2addr(path)) #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \ FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd) #define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \ FdRelease(((TsanInterceptorContext *) ctx)->thr, pc, fd) #define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) \ FdAccess(((TsanInterceptorContext *) ctx)->thr, pc, fd) #define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \ FdSocketAccept(((TsanInterceptorContext *) ctx)->thr, pc, fd, newfd) #define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \ ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name) #define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ if (pthread_equal(pthread_self(), reinterpret_cast(thread))) \ COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name); \ else \ __tsan::ctx->thread_registry.SetThreadNameByUserId(thread, name) #define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name) #define COMMON_INTERCEPTOR_ON_EXIT(ctx) \ OnExit(((TsanInterceptorContext *) ctx)->thr) #define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \ off) \ do { \ return mmap_interceptor(thr, pc, REAL(mmap), addr, sz, prot, flags, fd, \ off); \ } while (false) #define COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, sz) \ do { \ return munmap_interceptor(thr, pc, REAL(munmap), addr, sz); \ } while (false) #if !SANITIZER_APPLE #define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \ HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \ ((TsanInterceptorContext *)ctx)->pc, msg) #endif #define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \ if (TsanThread *t = GetCurrentThread()) { \ *begin = t->tls_begin(); \ *end = t->tls_end(); \ } else { \ *begin = *end = 0; \ } #define COMMON_INTERCEPTOR_USER_CALLBACK_START() \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START() #define COMMON_INTERCEPTOR_USER_CALLBACK_END() \ SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END() #include "sanitizer_common/sanitizer_common_interceptors.inc" static int sigaction_impl(int sig, const __sanitizer_sigaction *act, __sanitizer_sigaction *old); static __sanitizer_sighandler_ptr signal_impl(int sig, __sanitizer_sighandler_ptr h); #define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signo, act, oldact) \ { return sigaction_impl(signo, act, oldact); } #define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) \ { return (uptr)signal_impl(signo, (__sanitizer_sighandler_ptr)handler); } #define SIGNAL_INTERCEPTOR_ENTER() LazyInitialize(cur_thread_init()) #include "sanitizer_common/sanitizer_signal_interceptors.inc" int sigaction_impl(int sig, const __sanitizer_sigaction *act, __sanitizer_sigaction *old) { // Note: if we call REAL(sigaction) directly for any reason without proxying // the signal handler through sighandler, very bad things will happen. // The handler will run synchronously and corrupt tsan per-thread state. SCOPED_INTERCEPTOR_RAW(sigaction, sig, act, old); if (sig <= 0 || sig >= kSigCount) { errno = errno_EINVAL; return -1; } __sanitizer_sigaction *sigactions = interceptor_ctx()->sigactions; __sanitizer_sigaction old_stored; if (old) internal_memcpy(&old_stored, &sigactions[sig], sizeof(old_stored)); __sanitizer_sigaction newact; if (act) { // Copy act into sigactions[sig]. // Can't use struct copy, because compiler can emit call to memcpy. // Can't use internal_memcpy, because it copies byte-by-byte, // and signal handler reads the handler concurrently. It can read // some bytes from old value and some bytes from new value. // Use volatile to prevent insertion of memcpy. sigactions[sig].handler = *(volatile __sanitizer_sighandler_ptr const *)&act->handler; sigactions[sig].sa_flags = *(volatile int const *)&act->sa_flags; internal_memcpy(&sigactions[sig].sa_mask, &act->sa_mask, sizeof(sigactions[sig].sa_mask)); #if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD sigactions[sig].sa_restorer = act->sa_restorer; #endif internal_memcpy(&newact, act, sizeof(newact)); internal_sigfillset(&newact.sa_mask); if ((act->sa_flags & SA_SIGINFO) || ((uptr)act->handler != sig_ign && (uptr)act->handler != sig_dfl)) { newact.sa_flags |= SA_SIGINFO; newact.sigaction = sighandler; } ReleaseStore(thr, pc, (uptr)&sigactions[sig]); act = &newact; } int res = REAL(sigaction)(sig, act, old); if (res == 0 && old && old->sigaction == sighandler) internal_memcpy(old, &old_stored, sizeof(*old)); return res; } static __sanitizer_sighandler_ptr signal_impl(int sig, __sanitizer_sighandler_ptr h) { __sanitizer_sigaction act; act.handler = h; internal_memset(&act.sa_mask, -1, sizeof(act.sa_mask)); act.sa_flags = 0; __sanitizer_sigaction old; int res = sigaction_symname(sig, &act, &old); if (res) return (__sanitizer_sighandler_ptr)sig_err; return old.handler; } #define TSAN_SYSCALL() \ ThreadState *thr = cur_thread(); \ if (thr->ignore_interceptors) \ return; \ ScopedSyscall scoped_syscall(thr) struct ScopedSyscall { ThreadState *thr; explicit ScopedSyscall(ThreadState *thr) : thr(thr) { LazyInitialize(thr); } ~ScopedSyscall() { ProcessPendingSignals(thr); } }; #if !SANITIZER_FREEBSD && !SANITIZER_APPLE static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) { TSAN_SYSCALL(); MemoryAccessRange(thr, pc, p, s, write); } static USED void syscall_acquire(uptr pc, uptr addr) { TSAN_SYSCALL(); Acquire(thr, pc, addr); DPrintf("syscall_acquire(0x%zx))\n", addr); } static USED void syscall_release(uptr pc, uptr addr) { TSAN_SYSCALL(); DPrintf("syscall_release(0x%zx)\n", addr); Release(thr, pc, addr); } static void syscall_fd_close(uptr pc, int fd) { auto *thr = cur_thread(); FdClose(thr, pc, fd); } static USED void syscall_fd_acquire(uptr pc, int fd) { TSAN_SYSCALL(); FdAcquire(thr, pc, fd); DPrintf("syscall_fd_acquire(%d)\n", fd); } static USED void syscall_fd_release(uptr pc, int fd) { TSAN_SYSCALL(); DPrintf("syscall_fd_release(%d)\n", fd); FdRelease(thr, pc, fd); } static void syscall_pre_fork(uptr pc) { ForkBefore(cur_thread(), pc); } static void syscall_post_fork(uptr pc, int pid) { ThreadState *thr = cur_thread(); if (pid == 0) { // child ForkChildAfter(thr, pc, true); FdOnFork(thr, pc); } else if (pid > 0) { // parent ForkParentAfter(thr, pc); } else { // error ForkParentAfter(thr, pc); } } #endif #define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \ syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false) #define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \ syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), true) #define COMMON_SYSCALL_POST_READ_RANGE(p, s) \ do { \ (void)(p); \ (void)(s); \ } while (false) #define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \ do { \ (void)(p); \ (void)(s); \ } while (false) #define COMMON_SYSCALL_ACQUIRE(addr) \ syscall_acquire(GET_CALLER_PC(), (uptr)(addr)) #define COMMON_SYSCALL_RELEASE(addr) \ syscall_release(GET_CALLER_PC(), (uptr)(addr)) #define COMMON_SYSCALL_FD_CLOSE(fd) syscall_fd_close(GET_CALLER_PC(), fd) #define COMMON_SYSCALL_FD_ACQUIRE(fd) syscall_fd_acquire(GET_CALLER_PC(), fd) #define COMMON_SYSCALL_FD_RELEASE(fd) syscall_fd_release(GET_CALLER_PC(), fd) #define COMMON_SYSCALL_PRE_FORK() \ syscall_pre_fork(GET_CALLER_PC()) #define COMMON_SYSCALL_POST_FORK(res) \ syscall_post_fork(GET_CALLER_PC(), res) #include "sanitizer_common/sanitizer_common_syscalls.inc" #include "sanitizer_common/sanitizer_syscalls_netbsd.inc" #ifdef NEED_TLS_GET_ADDR static void handle_tls_addr(void *arg, void *res) { ThreadState *thr = cur_thread(); if (!thr) return; DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, res, thr->tls_addr, thr->tls_addr + thr->tls_size); if (!dtv) return; // New DTLS block has been allocated. MemoryResetRange(thr, 0, dtv->beg, dtv->size); } #if !SANITIZER_S390 // Define own interceptor instead of sanitizer_common's for three reasons: // 1. It must not process pending signals. // Signal handlers may contain MOVDQA instruction (see below). // 2. It must be as simple as possible to not contain MOVDQA. // 3. Sanitizer_common version uses COMMON_INTERCEPTOR_INITIALIZE_RANGE which // is empty for tsan (meant only for msan). // Note: __tls_get_addr can be called with mis-aligned stack due to: // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58066 // So the interceptor must work with mis-aligned stack, in particular, does not // execute MOVDQA with stack addresses. TSAN_INTERCEPTOR(void *, __tls_get_addr, void *arg) { void *res = REAL(__tls_get_addr)(arg); handle_tls_addr(arg, res); return res; } #else // SANITIZER_S390 TSAN_INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) { uptr res = __tls_get_offset_wrapper(arg, REAL(__tls_get_offset)); char *tp = static_cast(__builtin_thread_pointer()); handle_tls_addr(arg, res + tp); return res; } #endif #endif #if SANITIZER_NETBSD TSAN_INTERCEPTOR(void, _lwp_exit) { SCOPED_TSAN_INTERCEPTOR(_lwp_exit); DestroyThreadState(); REAL(_lwp_exit)(); } #define TSAN_MAYBE_INTERCEPT__LWP_EXIT TSAN_INTERCEPT(_lwp_exit) #else #define TSAN_MAYBE_INTERCEPT__LWP_EXIT #endif #if SANITIZER_FREEBSD TSAN_INTERCEPTOR(void, thr_exit, tid_t *state) { SCOPED_TSAN_INTERCEPTOR(thr_exit, state); DestroyThreadState(); REAL(thr_exit(state)); } #define TSAN_MAYBE_INTERCEPT_THR_EXIT TSAN_INTERCEPT(thr_exit) #else #define TSAN_MAYBE_INTERCEPT_THR_EXIT #endif TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_init, void *c, void *a) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_destroy, void *c) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_signal, void *c) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_broadcast, void *c) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_wait, void *c, void *m) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_init, void *m, void *a) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_destroy, void *m) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_lock, void *m) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_trylock, void *m) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_unlock, void *m) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_init, void *l, void *a) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_destroy, void *l) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_rdlock, void *l) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_tryrdlock, void *l) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_wrlock, void *l) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_trywrlock, void *l) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_unlock, void *l) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, once, void *o, void (*i)()) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, sigmask, int f, void *n, void *o) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_init, void *c, void *a) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_signal, void *c) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_broadcast, void *c) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_wait, void *c, void *m) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_destroy, void *c) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_init, void *m, void *a) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_destroy, void *m) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_lock, void *m) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_trylock, void *m) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_unlock, void *m) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_init, void *m, void *a) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_destroy, void *m) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_rdlock, void *m) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_tryrdlock, void *m) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_wrlock, void *m) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_trywrlock, void *m) TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_unlock, void *m) TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(int, once, void *o, void (*f)()) TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(int, sigsetmask, sigmask, int a, void *b, void *c) namespace __tsan { static void finalize(void *arg) { ThreadState *thr = cur_thread(); int status = Finalize(thr); // Make sure the output is not lost. FlushStreams(); if (status) Die(); } #if !SANITIZER_APPLE && !SANITIZER_ANDROID && !SANITIZER_NETBSD static void unreachable() { Report("FATAL: ThreadSanitizer: unreachable called\n"); Die(); } #endif // Define default implementation since interception of libdispatch is optional. SANITIZER_WEAK_ATTRIBUTE void InitializeLibdispatchInterceptors() {} void InitializeInterceptors() { #if !SANITIZER_APPLE // We need to setup it early, because functions like dlsym() can call it. REAL(memset) = internal_memset; REAL(memcpy) = internal_memcpy; #endif new(interceptor_ctx()) InterceptorContext(); InitializeCommonInterceptors(); InitializeSignalInterceptors(); InitializeLibdispatchInterceptors(); #if !SANITIZER_APPLE InitializeSetjmpInterceptors(); #endif TSAN_INTERCEPT(longjmp_symname); TSAN_INTERCEPT(siglongjmp_symname); #if SANITIZER_NETBSD TSAN_INTERCEPT(_longjmp); #endif TSAN_INTERCEPT(malloc); TSAN_INTERCEPT(__libc_memalign); TSAN_INTERCEPT(calloc); TSAN_INTERCEPT(realloc); TSAN_INTERCEPT(reallocarray); TSAN_INTERCEPT(free); TSAN_INTERCEPT(cfree); TSAN_INTERCEPT(munmap); TSAN_MAYBE_INTERCEPT_MEMALIGN; TSAN_INTERCEPT(valloc); TSAN_MAYBE_INTERCEPT_PVALLOC; TSAN_INTERCEPT(posix_memalign); TSAN_INTERCEPT(strcpy); TSAN_INTERCEPT(strncpy); TSAN_INTERCEPT(strdup); TSAN_INTERCEPT(pthread_create); TSAN_INTERCEPT(pthread_join); TSAN_INTERCEPT(pthread_detach); TSAN_INTERCEPT(pthread_exit); #if SANITIZER_LINUX TSAN_INTERCEPT(pthread_tryjoin_np); TSAN_INTERCEPT(pthread_timedjoin_np); #endif TSAN_INTERCEPT_VER(pthread_cond_init, PTHREAD_ABI_BASE); TSAN_INTERCEPT_VER(pthread_cond_signal, PTHREAD_ABI_BASE); TSAN_INTERCEPT_VER(pthread_cond_broadcast, PTHREAD_ABI_BASE); TSAN_INTERCEPT_VER(pthread_cond_wait, PTHREAD_ABI_BASE); TSAN_INTERCEPT_VER(pthread_cond_timedwait, PTHREAD_ABI_BASE); TSAN_INTERCEPT_VER(pthread_cond_destroy, PTHREAD_ABI_BASE); TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT; TSAN_INTERCEPT(pthread_mutex_init); TSAN_INTERCEPT(pthread_mutex_destroy); TSAN_INTERCEPT(pthread_mutex_lock); TSAN_INTERCEPT(pthread_mutex_trylock); TSAN_INTERCEPT(pthread_mutex_timedlock); TSAN_INTERCEPT(pthread_mutex_unlock); #if SANITIZER_GLIBC # if !__GLIBC_PREREQ(2, 34) TSAN_INTERCEPT(__pthread_mutex_lock); TSAN_INTERCEPT(__pthread_mutex_unlock); # endif #endif TSAN_INTERCEPT(pthread_spin_init); TSAN_INTERCEPT(pthread_spin_destroy); TSAN_INTERCEPT(pthread_spin_lock); TSAN_INTERCEPT(pthread_spin_trylock); TSAN_INTERCEPT(pthread_spin_unlock); TSAN_INTERCEPT(pthread_rwlock_init); TSAN_INTERCEPT(pthread_rwlock_destroy); TSAN_INTERCEPT(pthread_rwlock_rdlock); TSAN_INTERCEPT(pthread_rwlock_tryrdlock); TSAN_INTERCEPT(pthread_rwlock_timedrdlock); TSAN_INTERCEPT(pthread_rwlock_wrlock); TSAN_INTERCEPT(pthread_rwlock_trywrlock); TSAN_INTERCEPT(pthread_rwlock_timedwrlock); TSAN_INTERCEPT(pthread_rwlock_unlock); TSAN_INTERCEPT(pthread_barrier_init); TSAN_INTERCEPT(pthread_barrier_destroy); TSAN_INTERCEPT(pthread_barrier_wait); TSAN_INTERCEPT(pthread_once); TSAN_INTERCEPT(fstat); TSAN_MAYBE_INTERCEPT___FXSTAT; TSAN_MAYBE_INTERCEPT_FSTAT64; TSAN_MAYBE_INTERCEPT___FXSTAT64; TSAN_INTERCEPT(open); TSAN_MAYBE_INTERCEPT_OPEN64; TSAN_INTERCEPT(creat); TSAN_MAYBE_INTERCEPT_CREAT64; TSAN_INTERCEPT(dup); TSAN_INTERCEPT(dup2); TSAN_INTERCEPT(dup3); TSAN_MAYBE_INTERCEPT_EVENTFD; TSAN_MAYBE_INTERCEPT_SIGNALFD; TSAN_MAYBE_INTERCEPT_INOTIFY_INIT; TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1; TSAN_INTERCEPT(socket); TSAN_INTERCEPT(socketpair); TSAN_INTERCEPT(connect); TSAN_INTERCEPT(bind); TSAN_INTERCEPT(listen); TSAN_MAYBE_INTERCEPT_EPOLL; TSAN_INTERCEPT(close); TSAN_MAYBE_INTERCEPT___CLOSE; TSAN_MAYBE_INTERCEPT___RES_ICLOSE; TSAN_INTERCEPT(pipe); TSAN_INTERCEPT(pipe2); TSAN_INTERCEPT(unlink); TSAN_INTERCEPT(tmpfile); TSAN_MAYBE_INTERCEPT_TMPFILE64; TSAN_INTERCEPT(abort); TSAN_INTERCEPT(rmdir); TSAN_INTERCEPT(closedir); TSAN_INTERCEPT(sigsuspend); TSAN_INTERCEPT(sigblock); TSAN_INTERCEPT(sigsetmask); TSAN_INTERCEPT(pthread_sigmask); TSAN_INTERCEPT(raise); TSAN_INTERCEPT(kill); TSAN_INTERCEPT(pthread_kill); TSAN_INTERCEPT(sleep); TSAN_INTERCEPT(usleep); TSAN_INTERCEPT(nanosleep); TSAN_INTERCEPT(pause); TSAN_INTERCEPT(gettimeofday); TSAN_INTERCEPT(getaddrinfo); TSAN_INTERCEPT(fork); TSAN_INTERCEPT(vfork); #if SANITIZER_LINUX TSAN_INTERCEPT(clone); #endif #if !SANITIZER_ANDROID TSAN_INTERCEPT(dl_iterate_phdr); #endif TSAN_MAYBE_INTERCEPT_ON_EXIT; TSAN_INTERCEPT(__cxa_atexit); TSAN_INTERCEPT(_exit); #ifdef SANITIZER_NETBSD TSAN_INTERCEPT(atexit); #endif #ifdef NEED_TLS_GET_ADDR #if !SANITIZER_S390 TSAN_INTERCEPT(__tls_get_addr); #else TSAN_INTERCEPT(__tls_get_addr_internal); TSAN_INTERCEPT(__tls_get_offset); #endif #endif TSAN_MAYBE_INTERCEPT__LWP_EXIT; TSAN_MAYBE_INTERCEPT_THR_EXIT; #if !SANITIZER_APPLE && !SANITIZER_ANDROID && !SANITIZER_NETBSD // Need to setup it, because interceptors check that the function is resolved. // But atexit is emitted directly into the module, so can't be resolved. REAL(atexit) = (int(*)(void(*)()))unreachable; #endif #if !SANITIZER_NETBSD if (REAL(__cxa_atexit)(&finalize, 0, 0)) #else if (REAL(atexit)((void (*)())&finalize)) #endif { Printf("ThreadSanitizer: failed to setup atexit callback\n"); Die(); } if (pthread_atfork(atfork_prepare, atfork_parent, atfork_child)) { Printf("ThreadSanitizer: failed to setup atfork callbacks\n"); Die(); } #if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD if (pthread_key_create(&interceptor_ctx()->finalize_key, &thread_finalize)) { Printf("ThreadSanitizer: failed to create thread key\n"); Die(); } #endif TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_init); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_destroy); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_signal); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_broadcast); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_wait); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_init); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_destroy); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_lock); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_trylock); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_unlock); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_init); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_destroy); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_rdlock); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_tryrdlock); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_wrlock); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_trywrlock); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_unlock); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(once); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(sigmask); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_init); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_signal); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_broadcast); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_wait); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_destroy); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_init); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_destroy); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_lock); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_trylock); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_unlock); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_init); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_destroy); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_rdlock); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_tryrdlock); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_wrlock); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_trywrlock); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_unlock); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(once); TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(sigsetmask); FdInit(); } } // namespace __tsan // Invisible barrier for tests. // There were several unsuccessful iterations for this functionality: // 1. Initially it was implemented in user code using // REAL(pthread_barrier_wait). But pthread_barrier_wait is not supported on // MacOS. Futexes are linux-specific for this matter. // 2. Then we switched to atomics+usleep(10). But usleep produced parasitic // "as-if synchronized via sleep" messages in reports which failed some // output tests. // 3. Then we switched to atomics+sched_yield. But this produced tons of tsan- // visible events, which lead to "failed to restore stack trace" failures. // Note that no_sanitize_thread attribute does not turn off atomic interception // so attaching it to the function defined in user code does not help. // That's why we now have what we have. constexpr u32 kBarrierThreadBits = 10; constexpr u32 kBarrierThreads = 1 << kBarrierThreadBits; extern "C" { SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_init( atomic_uint32_t *barrier, u32 num_threads) { if (num_threads >= kBarrierThreads) { Printf("barrier_init: count is too large (%d)\n", num_threads); Die(); } // kBarrierThreadBits lsb is thread count, // the remaining are count of entered threads. atomic_store(barrier, num_threads, memory_order_relaxed); } static u32 barrier_epoch(u32 value) { return (value >> kBarrierThreadBits) / (value & (kBarrierThreads - 1)); } SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_wait( atomic_uint32_t *barrier) { u32 old = atomic_fetch_add(barrier, kBarrierThreads, memory_order_relaxed); u32 old_epoch = barrier_epoch(old); if (barrier_epoch(old + kBarrierThreads) != old_epoch) { FutexWake(barrier, (1 << 30)); return; } for (;;) { u32 cur = atomic_load(barrier, memory_order_relaxed); if (barrier_epoch(cur) != old_epoch) return; FutexWait(barrier, cur); } } } // extern "C" @ 1.5 log @merge GCC 14.3.0. @ text @d483 1 d486 3 d2835 1 a2835 1 #if !SANITIZER_APPLE && !SANITIZER_ANDROID d2997 3 d3013 1 a3013 1 #if !SANITIZER_APPLE && !SANITIZER_ANDROID d3019 6 a3024 1 if (REAL(__cxa_atexit)(&finalize, 0, 0)) { @ 1.4 log @libtsan: Work around large TLS alignment issue. PR lib/58349: tsan expects cacheline-aligned thread-local variables but ld.elf_so only supports pointer-aligned @ text @d38 4 a41 1 #if SANITIZER_FREEBSD || SANITIZER_MAC d82 4 d100 1 d112 1 a112 1 #if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD d119 1 a119 1 #if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD d129 1 a129 1 #if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD d136 3 d142 1 a142 1 #elif !SANITIZER_MAC d155 1 a155 1 #if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD a165 3 #define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \ (!cur_thread_init()->is_inited) a174 1 atomic_uintptr_t in_blocking_func; d181 14 d200 1 d212 1 a212 1 #if !SANITIZER_MAC && !SANITIZER_NETBSD d257 3 a259 1 ThreadSignalContext *ctx = (ThreadSignalContext*)thr->signal_ctx; d261 9 a269 3 ctx = (ThreadSignalContext*)MmapOrDie(sizeof(*ctx), "ThreadSignalContext"); MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx)); thr->signal_ctx = ctx; d271 1 a271 1 return ctx; d276 1 a276 1 : thr_(thr), in_ignored_lib_(false), ignoring_(false) { d278 10 d300 2 d332 5 d338 13 a350 13 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) #elif SANITIZER_NETBSD # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \ INTERCEPT_FUNCTION(__libc_##func) # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \ INTERCEPT_FUNCTION(__libc_thr_##func) #else # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver) # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) d364 2 a365 9 : thr(thr) , ctx(SigCtx(thr)) { for (;;) { atomic_store(&ctx->in_blocking_func, 1, memory_order_relaxed); if (atomic_load(&thr->pending_signals, memory_order_relaxed) == 0) break; atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed); ProcessPendingSignals(thr); } d376 1 a376 1 atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed); a379 1 ThreadSignalContext *ctx; d408 4 a411 1 static void at_exit_wrapper() { d423 3 a425 1 Acquire(cur_thread(), (uptr)0, (uptr)ctx); d427 1 d431 2 a432 2 static void cxa_at_exit_wrapper(void *arg) { Acquire(cur_thread(), 0, (uptr)arg); d434 2 d437 1 d451 1 a451 1 return setup_at_exit_wrapper(thr, pc, (void(*)())f, 0, 0); d459 1 a459 1 return setup_at_exit_wrapper(thr, pc, (void(*)())f, arg, dso); d467 1 d483 2 a484 1 res = REAL(__cxa_atexit)((void (*)(void *a))at_exit_wrapper, 0, 0); d490 1 a490 1 res = REAL(__cxa_atexit)(cxa_at_exit_wrapper, ctx, dso); d496 2 a497 2 #if !SANITIZER_MAC && !SANITIZER_NETBSD static void on_exit_wrapper(int status, void *arg) { a498 2 uptr pc = 0; Acquire(thr, pc, (uptr)arg); d500 2 d503 1 d514 1 d519 1 a519 1 int res = REAL(on_exit)(on_exit_wrapper, ctx); d552 1 a552 3 buf->in_blocking_func = sctx ? atomic_load(&sctx->in_blocking_func, memory_order_relaxed) : false; d568 1 a568 1 if (sctx) { d570 2 a571 3 atomic_store(&sctx->in_blocking_func, buf->in_blocking_func, memory_order_relaxed); } d585 1 a585 1 #if SANITIZER_MAC d589 1 a589 1 #else // SANITIZER_MAC a598 46 #define TSAN_INTERCEPTOR_SETJMP_(x) __interceptor_ ## x #define TSAN_INTERCEPTOR_SETJMP__(x) TSAN_INTERCEPTOR_SETJMP_(x) #define TSAN_INTERCEPTOR_SETJMP TSAN_INTERCEPTOR_SETJMP__(setjmp_symname) #define TSAN_INTERCEPTOR_SIGSETJMP TSAN_INTERCEPTOR_SETJMP__(sigsetjmp_symname) #define TSAN_STRING_SETJMP SANITIZER_STRINGIFY(setjmp_symname) #define TSAN_STRING_SIGSETJMP SANITIZER_STRINGIFY(sigsetjmp_symname) // Not called. Merely to satisfy TSAN_INTERCEPT(). extern "C" SANITIZER_INTERFACE_ATTRIBUTE int TSAN_INTERCEPTOR_SETJMP(void *env); extern "C" int TSAN_INTERCEPTOR_SETJMP(void *env) { CHECK(0); return 0; } // FIXME: any reason to have a separate declaration? extern "C" SANITIZER_INTERFACE_ATTRIBUTE int __interceptor__setjmp(void *env); extern "C" int __interceptor__setjmp(void *env) { CHECK(0); return 0; } extern "C" SANITIZER_INTERFACE_ATTRIBUTE int TSAN_INTERCEPTOR_SIGSETJMP(void *env); extern "C" int TSAN_INTERCEPTOR_SIGSETJMP(void *env) { CHECK(0); return 0; } #if !SANITIZER_NETBSD extern "C" SANITIZER_INTERFACE_ATTRIBUTE int __interceptor___sigsetjmp(void *env); extern "C" int __interceptor___sigsetjmp(void *env) { CHECK(0); return 0; } #endif extern "C" int setjmp_symname(void *env); extern "C" int _setjmp(void *env); extern "C" int sigsetjmp_symname(void *env); #if !SANITIZER_NETBSD extern "C" int __sigsetjmp(void *env); #endif d605 16 a620 1 #endif // SANITIZER_MAC d659 1 a659 1 #if !SANITIZER_MAC d800 3 a802 2 TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) { SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz); d804 1 a804 1 int res = REAL(munmap)(addr, sz); d818 1 a818 1 #if !SANITIZER_MAC d849 1 a849 1 #if !SANITIZER_MAC d902 2 a903 1 static void guard_release(ThreadState *thr, uptr pc, atomic_uint32_t *g) { d906 1 a906 1 u32 old = atomic_exchange(g, kGuardDone, memory_order_release); d921 1 a921 1 #if SANITIZER_MAC d936 1 a936 1 guard_release(thr, pc, g); d941 1 a941 1 atomic_store(g, kGuardInit, memory_order_relaxed); d956 2 a957 1 ThreadSignalContext *sctx = thr->signal_ctx; d959 1 a959 1 thr->signal_ctx = 0; d965 1 a965 1 #if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD d997 1 a997 1 #if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD d1105 1 a1105 1 #if !SANITIZER_MAC && !SANITIZER_ANDROID d1201 2 a1202 3 ThreadSignalContext *ctx = SigCtx(thr); CHECK_EQ(atomic_load(&ctx->in_blocking_func, memory_order_relaxed), 1); atomic_store(&ctx->in_blocking_func, 0, memory_order_relaxed); d1273 1 a1273 1 #if SANITIZER_MAC d1340 13 d1363 1 a1363 1 #if !SANITIZER_MAC d1374 38 a1411 1 #if !SANITIZER_MAC d1494 1 a1494 1 #if !SANITIZER_MAC d1524 1 a1524 1 #if !SANITIZER_MAC d1542 1 a1542 1 #if !SANITIZER_MAC d1576 1 a1576 1 if (SANITIZER_MAC) d1586 1 a1586 1 if (guard_acquire(thr, pc, a, !SANITIZER_MAC)) { d1588 1 a1588 1 guard_release(thr, pc, a); d1593 1 a1593 1 #if SANITIZER_LINUX && !SANITIZER_ANDROID d1606 2 a1607 2 #if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf); d1610 1 a1610 1 return REAL(fstat)(fd, buf); d1612 1 a1612 1 SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf); d1615 1 a1615 1 return REAL(__fxstat)(0, fd, buf); d1619 1 a1619 1 #if SANITIZER_LINUX && !SANITIZER_ANDROID d1631 1 a1631 1 #if SANITIZER_LINUX && !SANITIZER_ANDROID d1713 1 a1713 1 #if !SANITIZER_MAC d1738 2 a1739 3 SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags); if (fd >= 0) FdClose(thr, pc, fd); d1741 1 a1741 1 if (fd >= 0) d1818 2 a1819 2 SCOPED_TSAN_INTERCEPTOR(close, fd); if (fd >= 0) d1826 2 a1827 3 SCOPED_TSAN_INTERCEPTOR(__close, fd); if (fd >= 0) FdClose(thr, pc, fd); d1838 1 a1838 1 SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr); d1841 1 a1841 4 for (int i = 0; i < cnt; i++) { if (fds[i] > 0) FdClose(thr, pc, fds[i]); } d1857 1 a1857 1 #if !SANITIZER_MAC d1922 1 a1922 1 SCOPED_TSAN_INTERCEPTOR(closedir, dirp); d1953 2 a1954 1 if (op == EPOLL_CTL_ADD && epfd >= 0) d1956 1 d1982 28 a2009 6 #define TSAN_MAYBE_INTERCEPT_EPOLL \ TSAN_INTERCEPT(epoll_create); \ TSAN_INTERCEPT(epoll_create1); \ TSAN_INTERCEPT(epoll_ctl); \ TSAN_INTERCEPT(epoll_wait); \ TSAN_INTERCEPT(epoll_pwait) d2041 1 a2041 1 static void ReportErrnoSpoiling(ThreadState *thr, uptr pc) { d2048 1 d2058 1 d2115 1 a2115 1 ReportErrnoSpoiling(thr, pc); d2143 11 a2153 1 static bool is_sync_signal(ThreadSignalContext *sctx, int sig) { d2155 1 a2155 3 sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS || // If we are sending signal to ourselves, we must process it now. (sctx && sig == sctx->int_signal_send); d2166 1 a2166 1 const bool sync = is_sync_signal(sctx, sig); d2171 1 a2171 1 (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed))) { d2173 2 a2174 2 if (sctx && atomic_load(&sctx->in_blocking_func, memory_order_relaxed)) { atomic_store(&sctx->in_blocking_func, 0, memory_order_relaxed); d2176 1 a2176 1 atomic_store(&sctx->in_blocking_func, 1, memory_order_relaxed); d2234 2 a2235 1 if (tid == pthread_self()) { a2236 1 } d2238 1 a2238 1 if (tid == pthread_self()) { d2295 1 d2312 1 d2345 1 a2345 1 #if !SANITIZER_MAC && !SANITIZER_ANDROID d2356 1 a2356 1 return IsAppMem(addr) && *MemToShadow(addr) != kShadowRodata; d2397 1 a2397 6 struct TsanInterceptorContext { ThreadState *thr; const uptr pc; }; #if !SANITIZER_MAC a2417 1 #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) a2422 16 #define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \ MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \ ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \ true) #define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \ MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr, \ ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \ false) #define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \ TsanInterceptorContext _ctx = {thr, pc}; \ ctx = (void *)&_ctx; \ (void)ctx; d2440 1 a2440 1 if (fd >= 0) FdClose(thr, pc, fd); \ d2443 13 d2457 10 a2466 4 libignore()->OnLibraryLoaded(filename) #define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \ libignore()->OnLibraryUnloaded() d2492 5 a2496 2 #define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ __tsan::ctx->thread_registry.SetThreadNameByUserId(thread, name) a2502 20 #define COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m) \ MutexPreLock(((TsanInterceptorContext *)ctx)->thr, \ ((TsanInterceptorContext *)ctx)->pc, (uptr)m) #define COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m) \ MutexPostLock(((TsanInterceptorContext *)ctx)->thr, \ ((TsanInterceptorContext *)ctx)->pc, (uptr)m) #define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \ MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \ ((TsanInterceptorContext *)ctx)->pc, (uptr)m) #define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \ MutexRepair(((TsanInterceptorContext *)ctx)->thr, \ ((TsanInterceptorContext *)ctx)->pc, (uptr)m) #define COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m) \ MutexInvalidAccess(((TsanInterceptorContext *)ctx)->thr, \ ((TsanInterceptorContext *)ctx)->pc, (uptr)m) d2510 6 a2515 1 #if !SANITIZER_MAC d2548 2 d2570 1 a2570 1 // and signal handler reads the handler concurrently. It it can read d2578 1 a2578 1 #if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD d2625 1 a2625 1 #if !SANITIZER_FREEBSD && !SANITIZER_MAC d2644 1 a2644 1 TSAN_SYSCALL(); d2779 20 d2806 1 d2808 1 d2831 1 a2831 1 #if !SANITIZER_MAC && !SANITIZER_ANDROID d2842 1 a2842 1 #if !SANITIZER_MAC d2854 2 a2855 11 #if !SANITIZER_MAC // We can not use TSAN_INTERCEPT to get setjmp addr, // because it does &setjmp and setjmp is not present in some versions of libc. using __interception::InterceptFunction; InterceptFunction(TSAN_STRING_SETJMP, (uptr*)&REAL(setjmp_symname), 0, 0); InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); InterceptFunction(TSAN_STRING_SIGSETJMP, (uptr*)&REAL(sigsetjmp_symname), 0, 0); #if !SANITIZER_NETBSD InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); #endif d2901 1 d2904 7 d3006 1 a3006 1 #if !SANITIZER_MAC && !SANITIZER_ANDROID d3021 1 a3021 1 #if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD d3028 20 d3055 1 d3057 1 d3089 3 a3091 1 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_init( d3106 1 a3106 1 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_wait( d3121 2 @ 1.3 log @make this actually be GCC 12.3.0's libsanitizer. the libsanitizer we used with GCC 9 and GCC 10 was significantly ahead of the GCC 9 and GCC 10 provided versions. @ text @d1085 1 a1085 1 CHECK_EQ(thr, &cur_thread_placeholder); @ 1.2 log @revert sanitizer back to the version we were using with GCC 9, since that one was already newer than the GCC 10 version. @ text @d34 2 a56 4 #if SANITIZER_ANDROID #define mallopt(a, b) #endif d74 2 a75 1 #if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 d85 2 a98 3 #if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_NETBSD extern "C" int mallopt(int param, int value); #endif d136 1 d156 1 a156 1 (cur_thread_init(), !cur_thread()->is_inited) a160 1 bool sigaction; a167 1 atomic_uintptr_t have_pending_signals; d194 1 a194 1 BlockingMutex atexit_mu; d197 1 a197 3 InterceptorContext() : libignore(LINKER_INITIALIZED), AtExitStack() { } d246 2 a247 2 : thr_(thr), pc_(pc), in_ignored_lib_(false), ignoring_(false) { Initialize(thr); d252 2 a253 1 !thr_->in_ignored_lib && libignore()->IsIgnored(pc, &in_ignored_lib_); d263 1 a263 1 CheckNoLocks(thr_); d267 8 a274 8 void ScopedInterceptor::EnableIgnores() { if (ignoring_) { ThreadIgnoreBegin(thr_, pc_, /*save_stack=*/false); if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports++; if (in_ignored_lib_) { DCHECK(!thr_->in_ignored_lib); thr_->in_ignored_lib = true; } d278 8 a285 8 void ScopedInterceptor::DisableIgnores() { if (ignoring_) { ThreadIgnoreEnd(thr_, pc_); if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports--; if (in_ignored_lib_) { DCHECK(thr_->in_ignored_lib); thr_->in_ignored_lib = false; } d321 1 a321 1 if (atomic_load(&ctx->have_pending_signals, memory_order_relaxed) == 0) d373 1 a373 1 BlockingMutexLock l(&interceptor_ctx()->atexit_mu); d383 1 a383 1 InternalFree(ctx); d390 1 a390 1 InternalFree(ctx); d416 1 a416 1 AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx)); d429 4 a432 1 BlockingMutexLock l(&interceptor_ctx()->atexit_mu); d442 1 a442 1 ThreadIgnoreEnd(thr, pc); d453 1 a453 1 InternalFree(ctx); d460 1 a460 1 AtExitCtx *ctx = (AtExitCtx*)InternalAlloc(sizeof(AtExitCtx)); d468 1 a468 1 ThreadIgnoreEnd(thr, pc); d534 1 a534 4 extern "C" void __tsan_setjmp(uptr sp) { cur_thread_init(); SetJmp(cur_thread(), sp); } d654 3 d658 1 a658 1 SCOPED_TSAN_INTERCEPTOR(__libc_memalign, align, sz); d771 5 d844 47 d911 1 a911 14 OnPotentiallyBlockingRegionBegin(); auto on_exit = at_scope_exit(&OnPotentiallyBlockingRegionEnd); for (;;) { u32 cmp = atomic_load(g, memory_order_acquire); if (cmp == 0) { if (atomic_compare_exchange_strong(g, &cmp, 1<<16, memory_order_relaxed)) return 1; } else if (cmp == 1) { Acquire(thr, pc, (uptr)g); return 0; } else { internal_sched_yield(); } } d916 1 a916 2 Release(thr, pc, (uptr)g); atomic_store(g, 1, memory_order_release); d921 1 a921 1 atomic_store(g, 0, memory_order_relaxed); d931 5 a940 2 DTLS_Destroy(); cur_thread_finalize(); d963 3 a965 1 atomic_uintptr_t tid; a971 1 int tid = 0; d973 1 a973 2 cur_thread_init(); ThreadState *thr = cur_thread(); d983 1 a983 1 ThreadIgnoreEnd(thr, 0); d985 1 a985 2 while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0) internal_sched_yield(); d988 2 a989 2 ThreadStart(thr, tid, GetTid(), ThreadType::Regular); atomic_store(&p->tid, 0, memory_order_release); d1011 5 a1015 3 VPrintf(1, "ThreadSanitizer: starting new threads after multi-threaded " "fork is not supported (pid %d). Continuing because of " "die_after_fork=0, but you are on your own\n", internal_getpid()); d1030 1 a1030 1 atomic_store(&p.tid, 0, memory_order_relaxed); d1037 1 a1037 1 ThreadIgnoreEnd(thr, pc); d1040 2 a1041 2 int tid = ThreadCreate(thr, pc, *(uptr*)th, IsStateDetached(detached)); CHECK_NE(tid, 0); d1049 2 a1050 3 atomic_store(&p.tid, tid, memory_order_release); while (atomic_load(&p.tid, memory_order_acquire) != 0) internal_sched_yield(); d1059 1 a1059 1 int tid = ThreadTid(thr, pc, (uptr)th); d1062 1 a1062 1 ThreadIgnoreEnd(thr, pc); d1072 2 a1073 2 SCOPED_TSAN_INTERCEPTOR(pthread_detach, th); int tid = ThreadTid(thr, pc, (uptr)th); d1093 2 a1094 2 SCOPED_TSAN_INTERCEPTOR(pthread_tryjoin_np, th, ret); int tid = ThreadTid(thr, pc, (uptr)th); d1097 1 a1097 1 ThreadIgnoreEnd(thr, pc); d1107 2 a1108 2 SCOPED_TSAN_INTERCEPTOR(pthread_timedjoin_np, th, ret, abstime); int tid = ThreadTid(thr, pc, (uptr)th); d1111 1 a1111 1 ThreadIgnoreEnd(thr, pc); d1158 3 d1166 5 d1173 2 a1174 1 static void cond_mutex_unlock(CondMutexUnlockCtx *arg) { d1180 1 a1180 1 ThreadSignalContext *ctx = SigCtx(arg->thr); d1183 1 a1183 1 MutexPostLock(arg->thr, arg->pc, (uptr)arg->m, MutexFlagDoPreLockOnPostLock); d1185 2 a1186 2 arg->thr->ignore_interceptors--; arg->si->~ScopedInterceptor(); d1188 1 d1197 3 a1199 3 static int cond_wait(ThreadState *thr, uptr pc, ScopedInterceptor *si, int (*fn)(void *c, void *m, void *abstime), void *c, void *m, void *t) { a1201 1 CondMutexUnlockCtx arg = {si, thr, pc, m}; d1208 1 d1210 5 a1214 1 fn, c, m, t, (void (*)(void *arg))cond_mutex_unlock, &arg); d1224 3 a1226 3 return cond_wait(thr, pc, &si, (int (*)(void *c, void *m, void *abstime))REAL( pthread_cond_wait), cond, m, 0); d1232 4 a1235 2 return cond_wait(thr, pc, &si, REAL(pthread_cond_timedwait), cond, m, abstime); d1238 15 d1258 6 a1263 2 return cond_wait(thr, pc, &si, REAL(pthread_cond_timedwait_relative_np), cond, m, reltime); d1475 1 a1475 1 MemoryWrite(thr, pc, (uptr)b, kSizeLog1); d1482 1 a1482 1 MemoryWrite(thr, pc, (uptr)b, kSizeLog1); d1490 1 a1490 1 MemoryRead(thr, pc, (uptr)b, kSizeLog1); d1492 1 a1492 1 MemoryRead(thr, pc, (uptr)b, kSizeLog1); d1514 3 a1516 3 u32 v = atomic_load(a, memory_order_acquire); if (v == 0 && atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) { d1518 1 a1518 10 if (!thr->in_ignored_lib) Release(thr, pc, (uptr)o); atomic_store(a, 2, memory_order_release); } else { while (v != 2) { internal_sched_yield(); v = atomic_load(a, memory_order_acquire); } if (!thr->in_ignored_lib) Acquire(thr, pc, (uptr)o); d1573 6 a1578 2 TSAN_INTERCEPTOR(int, open, const char *name, int flags, int mode) { SCOPED_TSAN_INTERCEPTOR(open, name, flags, mode); d1580 1 a1580 1 int fd = REAL(open)(name, flags, mode); d1587 6 a1592 2 TSAN_INTERCEPTOR(int, open64, const char *name, int flags, int mode) { SCOPED_TSAN_INTERCEPTOR(open64, name, flags, mode); d1594 1 a1594 1 int fd = REAL(open64)(name, flags, mode); d1952 13 d1966 2 a1967 2 bool sigact, int sig, __sanitizer_siginfo *info, void *uctx) { d1973 1 a1973 1 // and interceptors, because otherwise we can miss syncronization and report d1978 7 d1990 1 d1998 3 a2000 2 volatile uptr pc = sigact ? (uptr)sigactions[sig].sigaction : (uptr)sigactions[sig].handler; d2002 4 a2005 4 if (sigact) ((__sanitizer_sigactionhandler_ptr)pc)(sig, info, uctx); else ((__sanitizer_sighandler_ptr)pc)(sig); d2013 1 d2022 3 a2024 12 if (flags()->report_bugs && !sync && sig != SIGTERM && errno != 99) { VarSizeStackTrace stack; // StackTrace::GetNestInstructionPc(pc) is used because return address is // expected, OutputReport() will undo this. ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack); ThreadRegistryLock l(ctx->thread_registry); ScopedReport rep(ReportTypeErrnoInSignal); if (!IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack)) { rep.AddStack(stack, true); OutputReport(thr, rep); } } d2028 2 a2029 1 void ProcessPendingSignals(ThreadState *thr) { d2031 1 a2031 2 if (sctx == 0 || atomic_load(&sctx->have_pending_signals, memory_order_relaxed) == 0) a2032 1 atomic_store(&sctx->have_pending_signals, 0, memory_order_relaxed); d2041 2 a2042 2 CallUserSignalHandler(thr, false, true, signal->sigaction, sig, &signal->siginfo, &signal->ctx); d2059 2 a2060 5 void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig, __sanitizer_siginfo *info, void *ctx) { cur_thread_init(); ThreadState *thr = cur_thread(); d2076 1 a2076 1 CallUserSignalHandler(thr, sync, true, sigact, sig, info, ctx); d2085 1 a2085 1 CallUserSignalHandler(thr, sync, acq, sigact, sig, info, ctx); d2096 3 a2098 6 signal->sigaction = sigact; if (info) internal_memcpy(&signal->siginfo, info, sizeof(*info)); if (ctx) internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx)); atomic_store(&sctx->have_pending_signals, 1, memory_order_relaxed); a2101 8 static void rtl_sighandler(int sig) { rtl_generic_sighandler(false, sig, 0, 0); } static void rtl_sigaction(int sig, __sanitizer_siginfo *info, void *ctx) { rtl_generic_sighandler(true, sig, info, ctx); } d2160 1 a2160 1 ThreadIgnoreEnd(thr, pc); d2168 8 d2177 17 a2193 19 int pid; { // On OS X, REAL(fork) can call intercepted functions (OSSpinLockLock), and // we'll assert in CheckNoLocks() unless we ignore interceptors. ScopedIgnoreInterceptors ignore; pid = REAL(fork)(fake); } if (pid == 0) { // child ForkChildAfter(thr, pc); FdOnFork(thr, pc); } else if (pid > 0) { // parent ForkParentAfter(thr, pc); } else { // error ForkParentAfter(thr, pc); } return pid; d2213 31 d2255 1 a2255 1 return IsAppMem(addr) && *(u64*)MemToShadow(addr) != kShadowRodata; a2297 1 const uptr caller_pc; d2319 1 d2325 2 d2338 5 a2342 5 #define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \ TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \ ctx = (void *)&_ctx; \ (void) ctx; d2346 1 a2346 1 TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \ d2348 1 a2348 1 (void) ctx; d2395 1 a2395 1 __tsan::ctx->thread_registry->SetThreadNameByUserId(thread, name) d2467 1 a2467 1 // the signal handler through rtl_sigaction, very bad things will happen. d2470 4 d2495 4 a2498 5 if ((uptr)act->handler != sig_ign && (uptr)act->handler != sig_dfl) { if (newact.sa_flags & SA_SIGINFO) newact.sigaction = rtl_sigaction; else newact.handler = rtl_sighandler; d2504 2 a2505 6 if (res == 0 && old) { uptr cb = (uptr)old->sigaction; if (cb == (uptr)rtl_sigaction || cb == (uptr)rtl_sighandler) { internal_memcpy(old, &old_stored, sizeof(*old)); } } d2521 1 a2521 1 #define TSAN_SYSCALL() \ d2523 3 a2525 4 if (thr->ignore_interceptors) \ return; \ ScopedSyscall scoped_syscall(thr) \ /**/ d2530 1 a2530 4 explicit ScopedSyscall(ThreadState *thr) : thr(thr) { Initialize(thr); } d2543 1 a2543 1 static void syscall_acquire(uptr pc, uptr addr) { d2546 1 a2546 1 DPrintf("syscall_acquire(%p)\n", addr); d2549 1 a2549 1 static void syscall_release(uptr pc, uptr addr) { d2551 1 a2551 1 DPrintf("syscall_release(%p)\n", addr); d2563 1 a2563 1 DPrintf("syscall_fd_acquire(%p)\n", fd); d2568 1 a2568 1 DPrintf("syscall_fd_release(%p)\n", fd); d2572 1 a2572 4 static void syscall_pre_fork(uptr pc) { TSAN_SYSCALL(); ForkBefore(thr, pc); } d2575 1 a2575 1 TSAN_SYSCALL(); d2578 1 a2578 1 ForkChildAfter(thr, pc); d2630 14 d2656 8 a2663 9 ThreadState *thr = cur_thread(); if (!thr) return res; DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, res, thr->tls_addr, thr->tls_addr + thr->tls_size); if (!dtv) return res; // New DTLS block has been allocated. MemoryResetRange(thr, 0, dtv->beg, dtv->size); d2667 1 a2737 6 // Instruct libc malloc to consume less memory. #if SANITIZER_LINUX mallopt(1, 0); // M_MXFAST mallopt(-3, 32*1024); // M_MMAP_THRESHOLD #endif d2796 2 d2875 3 d2886 1 d2888 4 d2907 4 d2955 8 a2962 5 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_init(u64 *barrier, u32 count) { if (count >= (1 << 8)) { Printf("barrier_init: count is too large (%d)\n", count); Die(); d2964 3 a2966 2 // 8 lsb is thread count, the remaining are count of entered threads. *barrier = count; d2969 12 a2980 4 extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_wait(u64 *barrier) { unsigned old = __atomic_fetch_add(barrier, 1 << 8, __ATOMIC_RELAXED); unsigned old_epoch = (old >> 8) / (old & 0xff); d2982 2 a2983 3 unsigned cur = __atomic_load_n(barrier, __ATOMIC_RELAXED); unsigned cur_epoch = (cur >> 8) / (cur & 0xff); if (cur_epoch != old_epoch) d2985 1 a2985 1 internal_sched_yield(); @ 1.1 log @Initial revision @ text @@ 1.1.1.1 log @initial import of GCC 10.3.0. main changes include: caveats: - ABI issue between c++14 and c++17 fixed - profile mode is removed from libstdc++ - -fno-common is now the default new features: - new flags -fallocation-dce, -fprofile-partial-training, -fprofile-reproducible, -fprofile-prefix-path, and -fanalyzer - many new compile and link time optimisations - enhanced drive optimisations - openacc 2.6 support - openmp 5.0 features - new warnings: -Wstring-compare and -Wzero-length-bounds - extended warnings: -Warray-bounds, -Wformat-overflow, -Wrestrict, -Wreturn-local-addr, -Wstringop-overflow, -Warith-conversion, -Wmismatched-tags, and -Wredundant-tags - some likely C2X features implemented - more C++20 implemented - many new arm & intel CPUs known hundreds of reported bugs are fixed. full list of changes can be found at: https://gcc.gnu.org/gcc-10/changes.html @ text @@ 1.1.1.2 log @initial import of GCC 12.3.0. major changes in GCC 11 included: - The default mode for C++ is now -std=gnu++17 instead of -std=gnu++14. - When building GCC itself, the host compiler must now support C++11, rather than C++98. - Some short options of the gcov tool have been renamed: -i to -j and -j to -H. - ThreadSanitizer improvements. - Introduce Hardware-assisted AddressSanitizer support. - For targets that produce DWARF debugging information GCC now defaults to DWARF version 5. This can produce up to 25% more compact debug information compared to earlier versions. - Many optimisations. - The existing malloc attribute has been extended so that it can be used to identify allocator/deallocator API pairs. A pair of new -Wmismatched-dealloc and -Wmismatched-new-delete warnings are added. - Other new warnings: -Wsizeof-array-div, enabled by -Wall, warns about divisions of two sizeof operators when the first one is applied to an array and the divisor does not equal the size of the array element. -Wstringop-overread, enabled by default, warns about calls to string functions reading past the end of the arrays passed to them as arguments. -Wtsan, enabled by default, warns about unsupported features in ThreadSanitizer (currently std::atomic_thread_fence). - Enchanced warnings: -Wfree-nonheap-object detects many more instances of calls to deallocation functions with pointers not returned from a dynamic memory allocation function. -Wmaybe-uninitialized diagnoses passing pointers or references to uninitialized memory to functions taking const-qualified arguments. -Wuninitialized detects reads from uninitialized dynamically allocated memory. -Warray-parameter warns about functions with inconsistent array forms. -Wvla-parameter warns about functions with inconsistent VLA forms. - Several new features from the upcoming C2X revision of the ISO C standard are supported with -std=c2x and -std=gnu2x. - Several C++20 features have been implemented. - The C++ front end has experimental support for some of the upcoming C++23 draft. - Several new C++ warnings. - Enhanced Arm, AArch64, x86, and RISC-V CPU support. - The implementation of how program state is tracked within -fanalyzer has been completely rewritten with many enhancements. see https://gcc.gnu.org/gcc-11/changes.html for a full list. major changes in GCC 12 include: - An ABI incompatibility between C and C++ when passing or returning by value certain aggregates containing zero width bit-fields has been discovered on various targets. x86-64, ARM and AArch64 will always ignore them (so there is a C ABI incompatibility between GCC 11 and earlier with GCC 12 or later), PowerPC64 ELFv2 always take them into account (so there is a C++ ABI incompatibility, GCC 4.4 and earlier compatible with GCC 12 or later, incompatible with GCC 4.5 through GCC 11). RISC-V has changed the handling of these already starting with GCC 10. As the ABI requires, MIPS takes them into account handling function return values so there is a C++ ABI incompatibility with GCC 4.5 through 11. - STABS: Support for emitting the STABS debugging format is deprecated and will be removed in the next release. All ports now default to emit DWARF (version 2 or later) debugging info or are obsoleted. - Vectorization is enabled at -O2 which is now equivalent to the original -O2 -ftree-vectorize -fvect-cost-model=very-cheap. - GCC now supports the ShadowCallStack sanitizer. - Support for __builtin_shufflevector compatible with the clang language extension was added. - Support for attribute unavailable was added. - Support for __builtin_dynamic_object_size compatible with the clang language extension was added. - New warnings: -Wbidi-chars warns about potentially misleading UTF-8 bidirectional control characters. -Warray-compare warns about comparisons between two operands of array type. - Some new features from the upcoming C2X revision of the ISO C standard are supported with -std=c2x and -std=gnu2x. - Several C++23 features have been implemented. - Many C++ enhancements across warnings and -f options. see https://gcc.gnu.org/gcc-12/changes.html for a full list. @ text @a33 2 #include d55 4 d76 1 a76 2 #if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 || \ defined(__s390x__) a85 2 extern "C" int pthread_atfork(void (*prepare)(void), void (*parent)(void), void (*child)(void)); d98 3 a137 1 typedef __sanitizer::u16 mode_t; d157 1 a157 1 (!cur_thread_init()->is_inited) d162 1 d170 1 d197 1 a197 1 Mutex atexit_mu; d200 3 a202 1 InterceptorContext() : libignore(LINKER_INITIALIZED), atexit_mu(MutexTypeAtExit), AtExitStack() {} d251 2 a252 2 : thr_(thr), in_ignored_lib_(false), ignoring_(false) { LazyInitialize(thr); d257 1 a257 2 !thr_->in_ignored_lib && (flags()->ignore_interceptors_accesses || libignore()->IsIgnored(pc, &in_ignored_lib_)); d267 1 a267 1 CheckedMutex::CheckNoLocks(); d271 8 a278 8 NOINLINE void ScopedInterceptor::EnableIgnoresImpl() { ThreadIgnoreBegin(thr_, 0); if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports++; if (in_ignored_lib_) { DCHECK(!thr_->in_ignored_lib); thr_->in_ignored_lib = true; d282 8 a289 8 NOINLINE void ScopedInterceptor::DisableIgnoresImpl() { ThreadIgnoreEnd(thr_); if (flags()->ignore_noninstrumented_modules) thr_->suppress_reports--; if (in_ignored_lib_) { DCHECK(thr_->in_ignored_lib); thr_->in_ignored_lib = false; d325 1 a325 1 if (atomic_load(&thr->pending_signals, memory_order_relaxed) == 0) d377 1 a377 1 Lock l(&interceptor_ctx()->atexit_mu); d387 1 a387 1 Free(ctx); d394 1 a394 1 Free(ctx); d420 1 a420 1 auto *ctx = New(); d433 1 a433 4 Lock l(&interceptor_ctx()->atexit_mu); // __cxa_atexit calls calloc. If we don't ignore interceptors, we will fail // due to atexit_mu held on exit from the calloc interceptor. ScopedIgnoreInterceptors ignore; d443 1 a443 1 ThreadIgnoreEnd(thr); d454 1 a454 1 Free(ctx); d461 1 a461 1 auto *ctx = New(); d469 1 a469 1 ThreadIgnoreEnd(thr); d535 4 a538 1 extern "C" void __tsan_setjmp(uptr sp) { SetJmp(cur_thread_init(), sp); } a657 3 // In glibc<2.25, dynamic TLS blocks are allocated by __libc_memalign. Intercept // __libc_memalign so that (1) we can detect races (2) free will not be called // on libc internally allocated blocks. d659 1 a659 1 SCOPED_INTERCEPTOR_RAW(__libc_memalign, align, sz); a771 5 if (!IsAppMem((uptr)res) || !IsAppMem((uptr)res + sz - 1)) { Report("ThreadSanitizer: mmap at bad address: addr=%p size=%p res=%p\n", addr, (void*)sz, res); Die(); } a839 47 // Both __cxa_guard_acquire and pthread_once 0-initialize // the object initially. pthread_once does not have any // other ABI requirements. __cxa_guard_acquire assumes // that any non-0 value in the first byte means that // initialization is completed. Contents of the remaining // bytes are up to us. constexpr u32 kGuardInit = 0; constexpr u32 kGuardDone = 1; constexpr u32 kGuardRunning = 1 << 16; constexpr u32 kGuardWaiter = 1 << 17; static int guard_acquire(ThreadState *thr, uptr pc, atomic_uint32_t *g, bool blocking_hooks = true) { if (blocking_hooks) OnPotentiallyBlockingRegionBegin(); auto on_exit = at_scope_exit([blocking_hooks] { if (blocking_hooks) OnPotentiallyBlockingRegionEnd(); }); for (;;) { u32 cmp = atomic_load(g, memory_order_acquire); if (cmp == kGuardInit) { if (atomic_compare_exchange_strong(g, &cmp, kGuardRunning, memory_order_relaxed)) return 1; } else if (cmp == kGuardDone) { if (!thr->in_ignored_lib) Acquire(thr, pc, (uptr)g); return 0; } else { if ((cmp & kGuardWaiter) || atomic_compare_exchange_strong(g, &cmp, cmp | kGuardWaiter, memory_order_relaxed)) FutexWait(g, cmp | kGuardWaiter); } } } static void guard_release(ThreadState *thr, uptr pc, atomic_uint32_t *g) { if (!thr->in_ignored_lib) Release(thr, pc, (uptr)g); u32 old = atomic_exchange(g, kGuardDone, memory_order_release); if (old & kGuardWaiter) FutexWake(g, 1 << 30); } d860 14 a873 1 return guard_acquire(thr, pc, g); d878 2 a879 1 guard_release(thr, pc, g); d884 1 a884 1 atomic_store(g, kGuardInit, memory_order_relaxed); a893 5 DTLS_Destroy(); cur_thread_finalize(); } void PlatformCleanUpThreadState(ThreadState *thr) { d899 2 d923 1 a923 3 Tid tid; Semaphore created; Semaphore started; d930 1 d932 2 a933 1 ThreadState *thr = cur_thread_init(); d943 1 a943 1 ThreadIgnoreEnd(thr); d945 2 a946 1 p->created.Wait(); d949 2 a950 2 ThreadStart(thr, p->tid, GetTid(), ThreadType::Regular); p->started.Post(); d972 3 a974 5 VPrintf(1, "ThreadSanitizer: starting new threads after multi-threaded " "fork is not supported (pid %lu). Continuing because of " "die_after_fork=0, but you are on your own\n", internal_getpid()); d989 1 a989 1 p.tid = kMainTid; d996 1 a996 1 ThreadIgnoreEnd(thr); d999 2 a1000 2 p.tid = ThreadCreate(thr, pc, *(uptr *)th, IsStateDetached(detached)); CHECK_NE(p.tid, kMainTid); d1008 3 a1010 2 p.created.Post(); p.started.Wait(); d1019 1 a1019 1 Tid tid = ThreadConsumeTid(thr, pc, (uptr)th); d1022 1 a1022 1 ThreadIgnoreEnd(thr); d1032 2 a1033 2 SCOPED_INTERCEPTOR_RAW(pthread_detach, th); Tid tid = ThreadConsumeTid(thr, pc, (uptr)th); d1053 2 a1054 2 SCOPED_INTERCEPTOR_RAW(pthread_tryjoin_np, th, ret); Tid tid = ThreadConsumeTid(thr, pc, (uptr)th); d1057 1 a1057 1 ThreadIgnoreEnd(thr); d1067 2 a1068 2 SCOPED_INTERCEPTOR_RAW(pthread_timedjoin_np, th, ret, abstime); Tid tid = ThreadConsumeTid(thr, pc, (uptr)th); d1071 1 a1071 1 ThreadIgnoreEnd(thr); a1117 3 namespace { template a1122 5 void *c; const Fn &fn; int Cancel() const { return fn(); } void Unlock() const; d1125 1 a1125 2 template void CondMutexUnlockCtx::Unlock() const { d1131 1 a1131 1 ThreadSignalContext *ctx = SigCtx(thr); d1134 1 a1134 1 MutexPostLock(thr, pc, (uptr)m, MutexFlagDoPreLockOnPostLock); d1136 2 a1137 2 thr->ignore_interceptors--; si->~ScopedInterceptor(); a1138 1 } // namespace d1147 3 a1149 3 template int cond_wait(ThreadState *thr, uptr pc, ScopedInterceptor *si, const Fn &fn, void *c, void *m) { d1152 1 a1158 1 CondMutexUnlockCtx arg = {si, thr, pc, m, c, fn}; d1160 1 a1160 5 [](void *arg) -> int { return ((const CondMutexUnlockCtx *)arg)->Cancel(); }, [](void *arg) { ((const CondMutexUnlockCtx *)arg)->Unlock(); }, &arg); d1170 3 a1172 3 return cond_wait( thr, pc, &si, [=]() { return REAL(pthread_cond_wait)(cond, m); }, cond, m); d1178 2 a1179 4 return cond_wait( thr, pc, &si, [=]() { return REAL(pthread_cond_timedwait)(cond, m, abstime); }, cond, m); a1181 15 #if SANITIZER_LINUX INTERCEPTOR(int, pthread_cond_clockwait, void *c, void *m, __sanitizer_clockid_t clock, void *abstime) { void *cond = init_cond(c); SCOPED_TSAN_INTERCEPTOR(pthread_cond_clockwait, cond, m, clock, abstime); return cond_wait( thr, pc, &si, [=]() { return REAL(pthread_cond_clockwait)(cond, m, clock, abstime); }, cond, m); } #define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT TSAN_INTERCEPT(pthread_cond_clockwait) #else #define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT #endif d1187 2 a1188 6 return cond_wait( thr, pc, &si, [=]() { return REAL(pthread_cond_timedwait_relative_np)(cond, m, reltime); }, cond, m); d1400 1 a1400 1 MemoryAccess(thr, pc, (uptr)b, 1, kAccessWrite); d1407 1 a1407 1 MemoryAccess(thr, pc, (uptr)b, 1, kAccessWrite); d1415 1 a1415 1 MemoryAccess(thr, pc, (uptr)b, 1, kAccessRead); d1417 1 a1417 1 MemoryAccess(thr, pc, (uptr)b, 1, kAccessRead); d1439 3 a1441 3 // Mac OS X appears to use pthread_once() where calling BlockingRegion hooks // result in crashes due to too little stack space. if (guard_acquire(thr, pc, a, !SANITIZER_MAC)) { d1443 10 a1452 1 guard_release(thr, pc, a); d1507 2 a1508 6 TSAN_INTERCEPTOR(int, open, const char *name, int oflag, ...) { va_list ap; va_start(ap, oflag); mode_t mode = va_arg(ap, int); va_end(ap); SCOPED_TSAN_INTERCEPTOR(open, name, oflag, mode); d1510 1 a1510 1 int fd = REAL(open)(name, oflag, mode); d1517 2 a1518 6 TSAN_INTERCEPTOR(int, open64, const char *name, int oflag, ...) { va_list ap; va_start(ap, oflag); mode_t mode = va_arg(ap, int); va_end(ap); SCOPED_TSAN_INTERCEPTOR(open64, name, oflag, mode); d1520 1 a1520 1 int fd = REAL(open64)(name, oflag, mode); a1877 13 static void ReportErrnoSpoiling(ThreadState *thr, uptr pc) { VarSizeStackTrace stack; // StackTrace::GetNestInstructionPc(pc) is used because return address is // expected, OutputReport() will undo this. ObtainCurrentStack(thr, StackTrace::GetNextInstructionPc(pc), &stack); ThreadRegistryLock l(&ctx->thread_registry); ScopedReport rep(ReportTypeErrnoInSignal); if (!IsFiredSuppression(ctx, ReportTypeErrnoInSignal, stack)) { rep.AddStack(stack, true); OutputReport(thr, rep); } } d1879 2 a1880 2 int sig, __sanitizer_siginfo *info, void *uctx) { d1886 1 a1886 1 // and interceptors, because otherwise we can miss synchronization and report a1890 7 // For symbolizer we only process SIGSEGVs synchronously // (bug in symbolizer or in tsan). But we want to reset // in_symbolizer to fail gracefully. Symbolizer and user code // use different memory allocators, so if we don't reset // in_symbolizer we can get memory allocated with one being // feed with another, which can cause more crashes. int in_symbolizer = thr->in_symbolizer; a1895 1 thr->in_symbolizer = 0; d1903 2 a1904 3 volatile uptr pc = (sigactions[sig].sa_flags & SA_SIGINFO) ? (uptr)sigactions[sig].sigaction : (uptr)sigactions[sig].handler; d1906 4 a1909 4 // The callback can be either sa_handler or sa_sigaction. // They have different signatures, but we assume that passing // additional arguments to sa_handler works and is harmless. ((__sanitizer_sigactionhandler_ptr)pc)(sig, info, uctx); a1916 1 thr->in_symbolizer = in_symbolizer; d1925 12 a1936 3 if (ShouldReport(thr, ReportTypeErrnoInSignal) && !sync && sig != SIGTERM && errno != 99) ReportErrnoSpoiling(thr, pc); d1940 1 a1940 2 void ProcessPendingSignalsImpl(ThreadState *thr) { atomic_store(&thr->pending_signals, 0, memory_order_relaxed); d1942 2 a1943 1 if (sctx == 0) d1945 1 d1954 2 a1955 2 CallUserSignalHandler(thr, false, true, sig, &signal->siginfo, &signal->ctx); d1972 5 a1976 2 void sighandler(int sig, __sanitizer_siginfo *info, void *ctx) { ThreadState *thr = cur_thread_init(); d1992 1 a1992 1 CallUserSignalHandler(thr, sync, true, sig, info, ctx); d2001 1 a2001 1 CallUserSignalHandler(thr, sync, acq, sig, info, ctx); d2012 6 a2017 3 internal_memcpy(&signal->siginfo, info, sizeof(*info)); internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx)); atomic_store(&thr->pending_signals, 1, memory_order_relaxed); d2021 8 d2087 1 a2087 1 ThreadIgnoreEnd(thr); a2094 8 return REAL(fork)(fake); } void atfork_prepare() { if (in_symbolizer()) return; ThreadState *thr = cur_thread(); const uptr pc = StackTrace::GetCurrentPc(); d2096 19 a2114 17 } void atfork_parent() { if (in_symbolizer()) return; ThreadState *thr = cur_thread(); const uptr pc = StackTrace::GetCurrentPc(); ForkParentAfter(thr, pc); } void atfork_child() { if (in_symbolizer()) return; ThreadState *thr = cur_thread(); const uptr pc = StackTrace::GetCurrentPc(); ForkChildAfter(thr, pc, true); FdOnFork(thr, pc); a2133 31 #if SANITIZER_LINUX TSAN_INTERCEPTOR(int, clone, int (*fn)(void *), void *stack, int flags, void *arg, int *parent_tid, void *tls, pid_t *child_tid) { SCOPED_INTERCEPTOR_RAW(clone, fn, stack, flags, arg, parent_tid, tls, child_tid); struct Arg { int (*fn)(void *); void *arg; }; auto wrapper = +[](void *p) -> int { auto *thr = cur_thread(); uptr pc = GET_CURRENT_PC(); // Start the background thread for fork, but not for clone. // For fork we did this always and it's known to work (or user code has // adopted). But if we do this for the new clone interceptor some code // (sandbox2) fails. So model we used to do for years and don't start the // background thread after clone. ForkChildAfter(thr, pc, false); FdOnFork(thr, pc); auto *arg = static_cast(p); return arg->fn(arg->arg); }; ForkBefore(thr, pc); Arg arg_wrapper = {fn, arg}; int pid = REAL(clone)(wrapper, stack, flags, &arg_wrapper, parent_tid, tls, child_tid); ForkParentAfter(thr, pc); return pid; } #endif d2145 1 a2145 1 return IsAppMem(addr) && *MemToShadow(addr) != kShadowRodata; d2188 1 a2209 1 #define SANITIZER_INTERCEPT_TLS_GET_OFFSET 1 a2214 2 #define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \ (INTERCEPT_FUNCTION_VER(name, ver) || INTERCEPT_FUNCTION(name)) d2226 5 a2230 5 #define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \ SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \ TsanInterceptorContext _ctx = {thr, pc}; \ ctx = (void *)&_ctx; \ (void)ctx; d2234 1 a2234 1 TsanInterceptorContext _ctx = {thr, pc}; \ d2236 1 a2236 1 (void)ctx; d2283 1 a2283 1 __tsan::ctx->thread_registry.SetThreadNameByUserId(thread, name) d2355 1 a2355 1 // the signal handler through sighandler, very bad things will happen. a2357 4 if (sig <= 0 || sig >= kSigCount) { errno = errno_EINVAL; return -1; } d2379 5 a2383 4 if ((act->sa_flags & SA_SIGINFO) || ((uptr)act->handler != sig_ign && (uptr)act->handler != sig_dfl)) { newact.sa_flags |= SA_SIGINFO; newact.sigaction = sighandler; d2389 6 a2394 2 if (res == 0 && old && old->sigaction == sighandler) internal_memcpy(old, &old_stored, sizeof(*old)); d2410 1 a2410 1 #define TSAN_SYSCALL() \ d2412 4 a2415 3 if (thr->ignore_interceptors) \ return; \ ScopedSyscall scoped_syscall(thr) d2420 4 a2423 1 explicit ScopedSyscall(ThreadState *thr) : thr(thr) { LazyInitialize(thr); } d2436 1 a2436 1 static USED void syscall_acquire(uptr pc, uptr addr) { d2439 1 a2439 1 DPrintf("syscall_acquire(0x%zx))\n", addr); d2442 1 a2442 1 static USED void syscall_release(uptr pc, uptr addr) { d2444 1 a2444 1 DPrintf("syscall_release(0x%zx)\n", addr); d2456 1 a2456 1 DPrintf("syscall_fd_acquire(%d)\n", fd); d2461 1 a2461 1 DPrintf("syscall_fd_release(%d)\n", fd); d2465 4 a2468 1 static void syscall_pre_fork(uptr pc) { ForkBefore(cur_thread(), pc); } d2471 1 a2471 1 ThreadState *thr = cur_thread(); d2474 1 a2474 1 ForkChildAfter(thr, pc, true); a2525 14 static void handle_tls_addr(void *arg, void *res) { ThreadState *thr = cur_thread(); if (!thr) return; DTLS::DTV *dtv = DTLS_on_tls_get_addr(arg, res, thr->tls_addr, thr->tls_addr + thr->tls_size); if (!dtv) return; // New DTLS block has been allocated. MemoryResetRange(thr, 0, dtv->beg, dtv->size); } #if !SANITIZER_S390 d2538 9 a2546 8 handle_tls_addr(arg, res); return res; } #else // SANITIZER_S390 TSAN_INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) { uptr res = __tls_get_offset_wrapper(arg, REAL(__tls_get_offset)); char *tp = static_cast(__builtin_thread_pointer()); handle_tls_addr(arg, res + tp); a2549 1 #endif d2620 6 a2683 2 TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT; a2760 3 #if SANITIZER_LINUX TSAN_INTERCEPT(clone); #endif a2768 1 #if !SANITIZER_S390 a2769 4 #else TSAN_INTERCEPT(__tls_get_addr_internal); TSAN_INTERCEPT(__tls_get_offset); #endif a2784 4 if (pthread_atfork(atfork_prepare, atfork_parent, atfork_child)) { Printf("ThreadSanitizer: failed to setup atfork callbacks\n"); Die(); } d2829 5 a2833 8 constexpr u32 kBarrierThreadBits = 10; constexpr u32 kBarrierThreads = 1 << kBarrierThreadBits; extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_init( atomic_uint32_t *barrier, u32 num_threads) { if (num_threads >= kBarrierThreads) { Printf("barrier_init: count is too large (%d)\n", num_threads); Die(); d2835 2 a2836 3 // kBarrierThreadBits lsb is thread count, // the remaining are count of entered threads. atomic_store(barrier, num_threads, memory_order_relaxed); d2839 4 a2842 12 static u32 barrier_epoch(u32 value) { return (value >> kBarrierThreadBits) / (value & (kBarrierThreads - 1)); } extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_wait( atomic_uint32_t *barrier) { u32 old = atomic_fetch_add(barrier, kBarrierThreads, memory_order_relaxed); u32 old_epoch = barrier_epoch(old); if (barrier_epoch(old + kBarrierThreads) != old_epoch) { FutexWake(barrier, (1 << 30)); return; } d2844 3 a2846 2 u32 cur = atomic_load(barrier, memory_order_relaxed); if (barrier_epoch(cur) != old_epoch) d2848 1 a2848 1 FutexWait(barrier, cur); @ 1.1.1.3 log @initial import of GCC 14.3.0. major changes in GCC 13: - improved sanitizer - zstd debug info compression - LTO improvements - SARIF based diagnostic support - new warnings: -Wxor-used-as-pow, -Wenum-int-mismatch, -Wself-move, -Wdangling-reference - many new -Wanalyzer* specific warnings - enhanced warnings: -Wpessimizing-move, -Wredundant-move - new attributes to mark file descriptors, c++23 "assume" - several C23 features added - several C++23 features added - many new features for Arm, x86, RISC-V major changes in GCC 14: - more strict C99 or newer support - ia64* marked deprecated (but seemingly still in GCC 15.) - several new hardening features - support for "hardbool", which can have user supplied values of true/false - explicit support for stack scrubbing upon function exit - better auto-vectorisation support - added clang-compatible __has_feature and __has_extension - more C23, including -std=c23 - several C++26 features added - better diagnostics in C++ templates - new warnings: -Wnrvo, Welaborated-enum-base - many new features for Arm, x86, RISC-V - possible ABI breaking change for SPARC64 and small structures with arrays of floats. @ text @d38 1 a38 4 DECLARE_REAL(void *, memcpy, void *to, const void *from, SIZE_T size) DECLARE_REAL(void *, memset, void *block, int c, SIZE_T size) #if SANITIZER_FREEBSD || SANITIZER_APPLE a78 4 #elif SANITIZER_LOONGARCH64 #define PTHREAD_ABI_BASE "GLIBC_2.36" #elif SANITIZER_RISCV64 # define PTHREAD_ABI_BASE "GLIBC_2.27" a92 1 extern "C" int pthread_equal(void *t1, void *t2); d104 1 a104 1 #if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD d111 1 a111 1 #if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD d121 1 a121 1 #if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_NETBSD a127 3 #if SANITIZER_HAS_SIGINFO const int SI_TIMER = -2; #endif d131 1 a131 1 #elif !SANITIZER_APPLE d144 1 a144 1 #if SANITIZER_FREEBSD || SANITIZER_APPLE || SANITIZER_NETBSD d155 3 d167 1 a173 14 void EnterBlockingFunc(ThreadState *thr) { for (;;) { // The order is important to not delay a signal infinitely if it's // delivered right before we set in_blocking_func. Note: we can't call // ProcessPendingSignals when in_blocking_func is set, or we can handle // a signal synchronously when we are already handling a signal. atomic_store(&thr->in_blocking_func, 1, memory_order_relaxed); if (atomic_load(&thr->pending_signals, memory_order_relaxed) == 0) break; atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed); ProcessPendingSignals(thr); } } a178 1 uptr pc; d190 1 a190 1 #if !SANITIZER_APPLE && !SANITIZER_NETBSD d235 1 a235 3 // This function may be called reentrantly if it is interrupted by a signal // handler. Use CAS to handle the race. uptr ctx = atomic_load(&thr->signal_ctx, memory_order_relaxed); d237 3 a239 9 uptr pctx = (uptr)MmapOrDie(sizeof(ThreadSignalContext), "ThreadSignalContext"); MemoryResetRange(thr, (uptr)&SigCtx, pctx, sizeof(ThreadSignalContext)); if (atomic_compare_exchange_strong(&thr->signal_ctx, &ctx, pctx, memory_order_relaxed)) { ctx = pctx; } else { UnmapOrDie((ThreadSignalContext *)pctx, sizeof(ThreadSignalContext)); } d241 1 a241 1 return (ThreadSignalContext *)ctx; d246 1 a246 1 : thr_(thr) { a247 10 if (UNLIKELY(atomic_load(&thr->in_blocking_func, memory_order_relaxed))) { // pthread_join is marked as blocking, but it's also known to call other // intercepted functions (mmap, free). If we don't reset in_blocking_func // we can get deadlocks and memory corruptions if we deliver a synchronous // signal inside of an mmap/free interceptor. // So reset it and restore it back in the destructor. // See https://github.com/google/sanitizers/issues/1540 atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed); in_blocking_func_ = true; } a259 2 if (UNLIKELY(in_blocking_func_)) EnterBlockingFunc(thr_); a289 5 #if SANITIZER_FREEBSD || SANITIZER_NETBSD # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func) #else # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver) #endif d291 13 a303 13 # define TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(func) \ INTERCEPT_FUNCTION(_pthread_##func) #else # define TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(func) #endif #if SANITIZER_NETBSD # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \ INTERCEPT_FUNCTION(__libc_##func) # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \ INTERCEPT_FUNCTION(__libc_thr_##func) #else # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) d317 9 a325 2 : thr(thr) { EnterBlockingFunc(thr); d336 1 a336 1 atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed); d340 1 d369 1 a369 4 // Note: we specifically call the function in such strange way // with "installed_at" because in reports it will appear between // callback frames and the frame that installed the callback. static void at_exit_callback_installed_at() { d381 1 a381 3 ThreadState *thr = cur_thread(); Acquire(thr, ctx->pc, (uptr)ctx); FuncEntry(thr, ctx->pc); a382 1 FuncExit(thr); d386 2 a387 2 static void cxa_at_exit_callback_installed_at(void *arg) { ThreadState *thr = cur_thread(); a388 2 Acquire(thr, ctx->pc, (uptr)arg); FuncEntry(thr, ctx->pc); a389 1 FuncExit(thr); d403 1 a403 1 return setup_at_exit_wrapper(thr, GET_CALLER_PC(), (void (*)())f, 0, 0); d411 1 a411 1 return setup_at_exit_wrapper(thr, GET_CALLER_PC(), (void (*)())f, arg, dso); a418 1 ctx->pc = pc; d434 1 a434 2 res = REAL(__cxa_atexit)((void (*)(void *a))at_exit_callback_installed_at, 0, 0); d440 1 a440 1 res = REAL(__cxa_atexit)(cxa_at_exit_callback_installed_at, ctx, dso); d446 2 a447 2 #if !SANITIZER_APPLE && !SANITIZER_NETBSD static void on_exit_callback_installed_at(int status, void *arg) { d449 2 a451 2 Acquire(thr, ctx->pc, (uptr)arg); FuncEntry(thr, ctx->pc); a452 1 FuncExit(thr); a462 1 ctx->pc = GET_CALLER_PC(); d467 1 a467 1 int res = REAL(on_exit)(on_exit_callback_installed_at, ctx); d500 3 a502 1 buf->in_blocking_func = atomic_load(&thr->in_blocking_func, memory_order_relaxed); d518 1 a518 1 if (sctx) d520 3 a522 2 atomic_store(&thr->in_blocking_func, buf->in_blocking_func, memory_order_relaxed); d536 1 a536 1 #if SANITIZER_APPLE d540 1 a540 1 #else // SANITIZER_APPLE d550 46 d602 1 a602 16 // The real interceptor for setjmp is special, and implemented in pure asm. We // just need to initialize the REAL functions so that they can be used in asm. static void InitializeSetjmpInterceptors() { // We can not use TSAN_INTERCEPT to get setjmp addr, because it does &setjmp and // setjmp is not present in some versions of libc. using __interception::InterceptFunction; InterceptFunction(SANITIZER_STRINGIFY(setjmp_symname), (uptr*)&REAL(setjmp_symname), 0, 0); InterceptFunction("_setjmp", (uptr*)&REAL(_setjmp), 0, 0); InterceptFunction(SANITIZER_STRINGIFY(sigsetjmp_symname), (uptr*)&REAL(sigsetjmp_symname), 0, 0); #if !SANITIZER_NETBSD InterceptFunction("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0); #endif } #endif // SANITIZER_APPLE d641 1 a641 1 #if !SANITIZER_APPLE d782 2 a783 3 template static int munmap_interceptor(ThreadState *thr, uptr pc, Munmap real_munmap, void *addr, SIZE_T sz) { d785 1 a785 1 int res = real_munmap(addr, sz); d799 1 a799 1 #if !SANITIZER_APPLE d830 1 a830 1 #if !SANITIZER_APPLE d883 1 a883 2 static void guard_release(ThreadState *thr, uptr pc, atomic_uint32_t *g, u32 v) { d886 1 a886 1 u32 old = atomic_exchange(g, v, memory_order_release); d901 1 a901 1 #if SANITIZER_APPLE d916 1 a916 1 guard_release(thr, pc, g, kGuardDone); d921 1 a921 1 guard_release(thr, pc, g, kGuardInit); d936 1 a936 2 ThreadSignalContext *sctx = (ThreadSignalContext *)atomic_load( &thr->signal_ctx, memory_order_relaxed); d938 1 a938 1 atomic_store(&thr->signal_ctx, 0, memory_order_relaxed); d944 1 a944 1 #if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD d976 1 a976 1 #if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD d1084 1 a1084 1 #if !SANITIZER_APPLE && !SANITIZER_ANDROID d1180 3 a1182 2 CHECK_EQ(atomic_load(&thr->in_blocking_func, memory_order_relaxed), 1); atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed); d1253 1 a1253 1 #if SANITIZER_APPLE a1319 13 TSAN_INTERCEPTOR(int, pthread_mutex_lock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_lock, m); MutexPreLock(thr, pc, (uptr)m); int res = REAL(pthread_mutex_lock)(m); if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); if (res == 0 || res == errno_EOWNERDEAD) MutexPostLock(thr, pc, (uptr)m); if (res == errno_EINVAL) MutexInvalidAccess(thr, pc, (uptr)m); return res; } d1330 1 a1330 1 #if !SANITIZER_APPLE d1341 1 a1341 38 TSAN_INTERCEPTOR(int, pthread_mutex_unlock, void *m) { SCOPED_TSAN_INTERCEPTOR(pthread_mutex_unlock, m); MutexUnlock(thr, pc, (uptr)m); int res = REAL(pthread_mutex_unlock)(m); if (res == errno_EINVAL) MutexInvalidAccess(thr, pc, (uptr)m); return res; } #if SANITIZER_GLIBC # if !__GLIBC_PREREQ(2, 34) // glibc 2.34 applies a non-default version for the two functions. They are no // longer expected to be intercepted by programs. TSAN_INTERCEPTOR(int, __pthread_mutex_lock, void *m) { SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_lock, m); MutexPreLock(thr, pc, (uptr)m); int res = REAL(__pthread_mutex_lock)(m); if (res == errno_EOWNERDEAD) MutexRepair(thr, pc, (uptr)m); if (res == 0 || res == errno_EOWNERDEAD) MutexPostLock(thr, pc, (uptr)m); if (res == errno_EINVAL) MutexInvalidAccess(thr, pc, (uptr)m); return res; } TSAN_INTERCEPTOR(int, __pthread_mutex_unlock, void *m) { SCOPED_TSAN_INTERCEPTOR(__pthread_mutex_unlock, m); MutexUnlock(thr, pc, (uptr)m); int res = REAL(__pthread_mutex_unlock)(m); if (res == errno_EINVAL) MutexInvalidAccess(thr, pc, (uptr)m); return res; } # endif #endif #if !SANITIZER_APPLE d1424 1 a1424 1 #if !SANITIZER_APPLE d1454 1 a1454 1 #if !SANITIZER_APPLE d1472 1 a1472 1 #if !SANITIZER_APPLE d1506 1 a1506 1 if (SANITIZER_APPLE) d1516 1 a1516 1 if (guard_acquire(thr, pc, a, !SANITIZER_APPLE)) { d1518 1 a1518 1 guard_release(thr, pc, a, kGuardDone); d1523 1 a1523 1 #if SANITIZER_GLIBC d1536 2 a1537 2 #if SANITIZER_GLIBC SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf); d1540 1 a1540 1 return REAL(__fxstat)(0, fd, buf); d1542 1 a1542 1 SCOPED_TSAN_INTERCEPTOR(fstat, fd, buf); d1545 1 a1545 1 return REAL(fstat)(fd, buf); d1549 1 a1549 1 #if SANITIZER_GLIBC d1561 1 a1561 1 #if SANITIZER_GLIBC d1643 1 a1643 1 #if !SANITIZER_APPLE d1668 3 a1670 2 SCOPED_INTERCEPTOR_RAW(signalfd, fd, mask, flags); FdClose(thr, pc, fd); d1672 1 a1672 1 if (!MustIgnoreInterceptor(thr)) d1749 2 a1750 2 SCOPED_INTERCEPTOR_RAW(close, fd); if (!in_symbolizer()) d1757 3 a1759 2 SCOPED_INTERCEPTOR_RAW(__close, fd); FdClose(thr, pc, fd); d1770 1 a1770 1 SCOPED_INTERCEPTOR_RAW(__res_iclose, state, free_addr); d1773 4 a1776 1 for (int i = 0; i < cnt; i++) FdClose(thr, pc, fds[i]); d1792 1 a1792 1 #if !SANITIZER_APPLE d1857 1 a1857 1 SCOPED_INTERCEPTOR_RAW(closedir, dirp); d1888 1 a1888 2 if (op == EPOLL_CTL_ADD && epfd >= 0) { FdPollAdd(thr, pc, epfd, fd); a1889 1 } d1915 6 a1920 28 TSAN_INTERCEPTOR(int, epoll_pwait2, int epfd, void *ev, int cnt, void *timeout, void *sigmask) { SCOPED_INTERCEPTOR_RAW(epoll_pwait2, epfd, ev, cnt, timeout, sigmask); // This function is new and may not be present in libc and/or kernel. // Since we effectively add it to libc (as will be probed by the program // using dlsym or a weak function pointer) we need to handle the case // when it's not present in the actual libc. if (!REAL(epoll_pwait2)) { errno = errno_ENOSYS; return -1; } if (MustIgnoreInterceptor(thr)) REAL(epoll_pwait2)(epfd, ev, cnt, timeout, sigmask); if (epfd >= 0) FdAccess(thr, pc, epfd); int res = BLOCK_REAL(epoll_pwait2)(epfd, ev, cnt, timeout, sigmask); if (res > 0 && epfd >= 0) FdAcquire(thr, pc, epfd); return res; } # define TSAN_MAYBE_INTERCEPT_EPOLL \ TSAN_INTERCEPT(epoll_create); \ TSAN_INTERCEPT(epoll_create1); \ TSAN_INTERCEPT(epoll_ctl); \ TSAN_INTERCEPT(epoll_wait); \ TSAN_INTERCEPT(epoll_pwait); \ TSAN_INTERCEPT(epoll_pwait2) d1952 1 a1952 1 static void ReportErrnoSpoiling(ThreadState *thr, uptr pc, int sig) { a1958 1 rep.SetSigNum(sig); a1967 1 CHECK(thr->slot); d2024 1 a2024 1 ReportErrnoSpoiling(thr, pc, sig); d2052 1 a2052 11 static bool is_sync_signal(ThreadSignalContext *sctx, int sig, __sanitizer_siginfo *info) { // If we are sending signal to ourselves, we must process it now. if (sctx && sig == sctx->int_signal_send) return true; #if SANITIZER_HAS_SIGINFO // POSIX timers can be configured to send any kind of signal; however, it // doesn't make any sense to consider a timer signal as synchronous! if (info->si_code == SI_TIMER) return false; #endif d2054 3 a2056 1 sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS; d2067 1 a2067 1 const bool sync = is_sync_signal(sctx, sig, info); d2072 1 a2072 1 atomic_load(&thr->in_blocking_func, memory_order_relaxed)) { d2074 2 a2075 2 if (atomic_load(&thr->in_blocking_func, memory_order_relaxed)) { atomic_store(&thr->in_blocking_func, 0, memory_order_relaxed); d2077 1 a2077 1 atomic_store(&thr->in_blocking_func, 1, memory_order_relaxed); d2135 1 a2135 2 bool self = pthread_equal(tid, pthread_self()); if (self) d2137 1 d2139 1 a2139 1 if (self) { a2195 1 #if !SANITIZER_IOS a2211 1 #endif d2244 1 a2244 1 #if !SANITIZER_APPLE && !SANITIZER_ANDROID d2255 1 a2255 1 return IsAppMem(addr) && *MemToShadow(addr) != Shadow::kRodata; d2296 6 a2301 1 #if !SANITIZER_APPLE d2322 1 d2328 16 d2361 1 a2361 1 FdClose(thr, pc, fd); \ a2363 13 #define COMMON_INTERCEPTOR_DLOPEN(filename, flag) \ ({ \ CheckNoDeepBind(filename, flag); \ ThreadIgnoreBegin(thr, 0); \ void *res = REAL(dlopen)(filename, flag); \ ThreadIgnoreEnd(thr); \ res; \ }) // Ignore interceptors in OnLibraryLoaded()/Unloaded(). These hooks use code // (ListOfModules::init, MemoryMappingLayout::DumpListOfModules) that make // intercepted calls, which can cause deadlockes with ReportRace() which also // uses this code. d2365 4 a2368 10 ({ \ ScopedIgnoreInterceptors ignore_interceptors; \ libignore()->OnLibraryLoaded(filename); \ }) #define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \ ({ \ ScopedIgnoreInterceptors ignore_interceptors; \ libignore()->OnLibraryUnloaded(); \ }) d2394 2 a2395 5 #define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \ if (pthread_equal(pthread_self(), reinterpret_cast(thread))) \ COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name); \ else \ __tsan::ctx->thread_registry.SetThreadNameByUserId(thread, name) d2402 20 d2429 1 a2429 6 #define COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, sz) \ do { \ return munmap_interceptor(thr, pc, REAL(munmap), addr, sz); \ } while (false) #if !SANITIZER_APPLE a2461 2 #define SIGNAL_INTERCEPTOR_ENTER() LazyInitialize(cur_thread_init()) d2482 1 a2482 1 // and signal handler reads the handler concurrently. It can read d2490 1 a2490 1 #if !SANITIZER_FREEBSD && !SANITIZER_APPLE && !SANITIZER_NETBSD d2537 1 a2537 1 #if !SANITIZER_FREEBSD && !SANITIZER_APPLE d2556 1 a2556 1 auto *thr = cur_thread(); a2690 20 TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_init, void *c, void *a) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_destroy, void *c) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_signal, void *c) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_broadcast, void *c) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, cond_wait, void *c, void *m) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_init, void *m, void *a) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_destroy, void *m) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_lock, void *m) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_trylock, void *m) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, mutex_unlock, void *m) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_init, void *l, void *a) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_destroy, void *l) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_rdlock, void *l) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_tryrdlock, void *l) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_wrlock, void *l) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_trywrlock, void *l) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, rwlock_unlock, void *l) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, once, void *o, void (*i)()) TSAN_INTERCEPTOR_FREEBSD_ALIAS(int, sigmask, int f, void *n, void *o) a2697 1 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_lock, void *m) a2698 1 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_unlock, void *m) d2721 1 a2721 1 #if !SANITIZER_APPLE && !SANITIZER_ANDROID d2732 1 a2732 1 #if !SANITIZER_APPLE d2744 11 a2754 2 #if !SANITIZER_APPLE InitializeSetjmpInterceptors(); a2799 1 TSAN_INTERCEPT(pthread_mutex_lock); a2801 7 TSAN_INTERCEPT(pthread_mutex_unlock); #if SANITIZER_GLIBC # if !__GLIBC_PREREQ(2, 34) TSAN_INTERCEPT(__pthread_mutex_lock); TSAN_INTERCEPT(__pthread_mutex_unlock); # endif #endif d2897 1 a2897 1 #if !SANITIZER_APPLE && !SANITIZER_ANDROID d2912 1 a2912 1 #if !SANITIZER_APPLE && !SANITIZER_NETBSD && !SANITIZER_FREEBSD a2918 20 TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_init); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_destroy); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_signal); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_broadcast); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(cond_wait); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_init); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_destroy); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_lock); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_trylock); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(mutex_unlock); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_init); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_destroy); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_rdlock); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_tryrdlock); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_wrlock); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_trywrlock); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(rwlock_unlock); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(once); TSAN_MAYBE_INTERCEPT_FREEBSD_ALIAS(sigmask); a2925 1 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_lock); a2926 1 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_unlock); d2958 1 a2958 3 extern "C" { SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_init( d2973 1 a2973 1 SANITIZER_INTERFACE_ATTRIBUTE void __tsan_testonly_barrier_wait( a2987 2 } // extern "C" @