tsan: add fiber support

This patch adds functions for managing fibers:

__tsan_get_current_fiber()
__tsan_create_fiber()
__tsan_destroy_fiber()
__tsan_switch_to_fiber()
__tsan_set_fiber_name()

See the added tests for use examples.

Author: yuri (Yuri Per)
Reviewed in: https://reviews.llvm.org/D54889

[The previous commit of this change was reverted,
this is a resubmit with a squashed fix for check_analyze.sh
and COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED]

llvm-svn: 353947

Author: dvyukov

compiler-rt/include/sanitizer/tsan_interface.h

@@ -136,6 +136,24 @@ void __tsan_external_assign_tag(void *addr, void *tag);
 void __tsan_external_read(void *addr, void *caller_pc, void *tag);
 void __tsan_external_write(void *addr, void *caller_pc, void *tag);
 
+// Fiber switching API.
+//   - TSAN context for fiber can be created by __tsan_create_fiber
+//     and freed by __tsan_destroy_fiber.
+//   - TSAN context of current fiber or thread can be obtained
+//     by calling __tsan_get_current_fiber.
+//   - __tsan_switch_to_fiber should be called immediatly before switch
+//     to fiber, such as call of swapcontext.
+//   - Fiber name can be set by __tsan_set_fiber_name.
+void *__tsan_get_current_fiber(void);
+void *__tsan_create_fiber(unsigned flags);
+void __tsan_destroy_fiber(void *fiber);
+void __tsan_switch_to_fiber(void *fiber, unsigned flags);
+void __tsan_set_fiber_name(void *fiber, const char *name);
+
+// Flags for __tsan_switch_to_fiber:
+// Do not establish a happens-before relation between fibers
+const unsigned __tsan_switch_to_fiber_no_sync = 1 << 0;
+
 #ifdef __cplusplus
 }  // extern "C"
 #endif

compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.h

@@ -31,6 +31,7 @@ enum ThreadStatus {
 enum class ThreadType {
   Regular, // Normal thread
   Worker,  // macOS Grand Central Dispatch (GCD) worker thread
+  Fiber,   // Fiber
 };
 
 // Generic thread context. Specific sanitizer tools may inherit from it.

compiler-rt/lib/tsan/check_analyze.sh

@@ -35,18 +35,12 @@ check() {
 }
 
 for f in write1 write2 write4 write8; do
-  check $f rsp 1
-  check $f push 1
-  check $f pop 8
-done
-
-for f in read1; do
   check $f rsp 1
   check $f push 2
   check $f pop 16
 done
 
-for f in read2 read4 read8; do
+for f in read1 read2 read4 read8; do
   check $f rsp 1
   check $f push 3
   check $f pop 24

compiler-rt/lib/tsan/rtl/tsan_interceptors.cc

@@ -153,7 +153,7 @@ const int SIG_SETMASK = 2;
 #endif
 
 #define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \
-  (!cur_thread()->is_inited)
+  (cur_thread_init(), !cur_thread()->is_inited)
 
 namespace __tsan {
 struct SignalDesc {
@@ -554,6 +554,7 @@ static void LongJmp(ThreadState *thr, uptr *env) {
 
 // FIXME: put everything below into a common extern "C" block?
 extern "C" void __tsan_setjmp(uptr sp, uptr mangled_sp) {
+  cur_thread_init();
   SetJmp(cur_thread(), sp, mangled_sp);
 }
 
@@ -942,6 +943,7 @@ extern "C" void *__tsan_thread_start_func(void *arg) {
   void *param = p->param;
   int tid = 0;
   {
+    cur_thread_init();
     ThreadState *thr = cur_thread();
     // Thread-local state is not initialized yet.
     ScopedIgnoreInterceptors ignore;
@@ -1053,6 +1055,9 @@ TSAN_INTERCEPTOR(int, pthread_detach, void *th) {
 TSAN_INTERCEPTOR(void, pthread_exit, void *retval) {
   {
     SCOPED_INTERCEPTOR_RAW(pthread_exit, retval);
+#if !SANITIZER_MAC && !SANITIZER_ANDROID
+    CHECK_EQ(thr, &cur_thread_placeholder);
+#endif
   }
   REAL(pthread_exit)(retval);
 }
@@ -1981,6 +1986,7 @@ static bool is_sync_signal(ThreadSignalContext *sctx, int sig) {
 void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig,
                                           __sanitizer_siginfo *info,
                                           void *ctx) {
+  cur_thread_init();
   ThreadState *thr = cur_thread();
   ThreadSignalContext *sctx = SigCtx(thr);
   if (sig < 0 || sig >= kSigCount) {

compiler-rt/lib/tsan/rtl/tsan_interceptors.h

@@ -23,13 +23,15 @@ LibIgnore *libignore();
 
 #if !SANITIZER_GO
 INLINE bool in_symbolizer() {
+  cur_thread_init();
   return UNLIKELY(cur_thread()->in_symbolizer);
 }
 #endif
 
 }  // namespace __tsan
 
 #define SCOPED_INTERCEPTOR_RAW(func, ...) \
+    cur_thread_init(); \
     ThreadState *thr = cur_thread(); \
     const uptr caller_pc = GET_CALLER_PC(); \
     ScopedInterceptor si(thr, #func, caller_pc); \

compiler-rt/lib/tsan/rtl/tsan_interface.cc

@@ -24,6 +24,7 @@ typedef u32 uint32_t;
 typedef u64 uint64_t;
 
 void __tsan_init() {
+  cur_thread_init();
   Initialize(cur_thread());
 }
 
@@ -123,6 +124,33 @@ void __sanitizer_unaligned_store64(uu64 *addr, u64 v) {
   __tsan_unaligned_write8(addr);
   *addr = v;
 }
+
+#if !SANITIZER_MAC && !SANITIZER_ANDROID
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_get_current_fiber() {
+  return cur_thread();
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void *__tsan_create_fiber(unsigned flags) {
+  return FiberCreate(cur_thread(), CALLERPC, flags);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_destroy_fiber(void *fiber) {
+  FiberDestroy(cur_thread(), CALLERPC, static_cast<ThreadState *>(fiber));
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_switch_to_fiber(void *fiber, unsigned flags) {
+  FiberSwitch(cur_thread(), CALLERPC, static_cast<ThreadState *>(fiber), flags);
+}
+
+SANITIZER_INTERFACE_ATTRIBUTE
+void __tsan_set_fiber_name(void *fiber, const char *name) {
+  ThreadSetName(static_cast<ThreadState *>(fiber), name);
+}
+#endif // !SANITIZER_MAC && !SANITIZER_ANDROID
 }  // extern "C"
 
 void __tsan_acquire(void *addr) {

compiler-rt/lib/tsan/rtl/tsan_rtl.h

@@ -384,6 +384,9 @@ struct ThreadState {
   // taken by epoch between synchs.
   // This way we can save one load from tls.
   u64 fast_synch_epoch;
+  // Technically `current` should be a separate THREADLOCAL variable;
+  // but it is placed here in order to share cache line with previous fields.
+  ThreadState* current;
   // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
   // We do not distinguish beteween ignoring reads and writes
   // for better performance.
@@ -462,11 +465,20 @@ struct ThreadState {
 #if SANITIZER_MAC || SANITIZER_ANDROID
 ThreadState *cur_thread();
 void cur_thread_finalize();
+INLINE void cur_thread_init() { }
 #else
 __attribute__((tls_model("initial-exec")))
 extern THREADLOCAL char cur_thread_placeholder[];
 INLINE ThreadState *cur_thread() {
-  return reinterpret_cast<ThreadState *>(&cur_thread_placeholder);
+  return reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current;
+}
+INLINE void cur_thread_init() {
+  ThreadState *thr = reinterpret_cast<ThreadState *>(cur_thread_placeholder);
+  if (UNLIKELY(!thr->current))
+    thr->current = thr;
+}
+INLINE void set_cur_thread(ThreadState *thr) {
+  reinterpret_cast<ThreadState *>(cur_thread_placeholder)->current = thr;
 }
 INLINE void cur_thread_finalize() { }
 #endif  // SANITIZER_MAC || SANITIZER_ANDROID
@@ -868,6 +880,16 @@ uptr ALWAYS_INLINE HeapEnd() {
 }
 #endif
 
+ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags);
+void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber);
+void FiberSwitch(ThreadState *thr, uptr pc, ThreadState *fiber, unsigned flags);
+
+// These need to match __tsan_switch_to_fiber_* flags defined in
+// tsan_interface.h. See documentation there as well.
+enum FiberSwitchFlags {
+  FiberSwitchFlagNoSync = 1 << 0, // __tsan_switch_to_fiber_no_sync
+};
+
 }  // namespace __tsan
 
 #endif  // TSAN_RTL_H

compiler-rt/lib/tsan/rtl/tsan_rtl_thread.cc

@@ -246,7 +246,8 @@ void ThreadStart(ThreadState *thr, int tid, tid_t os_id,
   uptr tls_addr = 0;
   uptr tls_size = 0;
 #if !SANITIZER_GO
-  GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size);
+  if (thread_type != ThreadType::Fiber)
+    GetThreadStackAndTls(tid == 0, &stk_addr, &stk_size, &tls_addr, &tls_size);
 
   if (tid) {
     if (stk_addr && stk_size)
@@ -404,4 +405,40 @@ void MemoryAccessRange(ThreadState *thr, uptr pc, uptr addr,
   }
 }
 
+#if !SANITIZER_MAC && !SANITIZER_ANDROID && !SANITIZER_GO
+void FiberSwitchImpl(ThreadState *from, ThreadState *to) {
+  Processor *proc = from->proc();
+  ProcUnwire(proc, from);
+  ProcWire(proc, to);
+  set_cur_thread(to);
+}
+
+ThreadState *FiberCreate(ThreadState *thr, uptr pc, unsigned flags) {
+  void *mem = internal_alloc(MBlockThreadContex, sizeof(ThreadState));
+  ThreadState *fiber = static_cast<ThreadState *>(mem);
+  internal_memset(fiber, 0, sizeof(*fiber));
+  int tid = ThreadCreate(thr, pc, 0, true);
+  FiberSwitchImpl(thr, fiber);
+  ThreadStart(fiber, tid, 0, ThreadType::Fiber);
+  FiberSwitchImpl(fiber, thr);
+  return fiber;
+}
+
+void FiberDestroy(ThreadState *thr, uptr pc, ThreadState *fiber) {
+  FiberSwitchImpl(thr, fiber);
+  ThreadFinish(fiber);
+  FiberSwitchImpl(fiber, thr);
+  internal_free(fiber);
+}
+
+void FiberSwitch(ThreadState *thr, uptr pc,
+                 ThreadState *fiber, unsigned flags) {
+  if (!(flags & FiberSwitchFlagNoSync))
+    Release(thr, pc, (uptr)fiber);
+  FiberSwitchImpl(thr, fiber);
+  if (!(flags & FiberSwitchFlagNoSync))
+    Acquire(fiber, pc, (uptr)fiber);
+}
+#endif
+
 }  // namespace __tsan

compiler-rt/test/tsan/fiber_asm.cc

@@ -0,0 +1,86 @@
+// RUN: %clang_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s
+// REQUIRES: x86_64-target-arch
+// UNSUPPORTED: darwin
+#include "test.h"
+
+struct ucontext {
+  void *sp;
+  void *fiber;
+};
+
+extern "C" {
+  void ucontext_do_switch(void **save, void **load);
+  void ucontext_trampoline();
+}
+
+__asm__(".global ucontext_do_switch\n"
+        "ucontext_do_switch:\n\t"
+        "pushq %rbp\n\t"
+        "pushq %r15\n\t"
+        "pushq %r14\n\t"
+        "pushq %r13\n\t"
+        "pushq %r12\n\t"
+        "pushq %rbx\n\t"
+        "movq %rsp, (%rdi)\n\t"
+        "movq (%rsi), %rsp\n\t"
+        "popq %rbx\n\t"
+        "popq %r12\n\t"
+        "popq %r13\n\t"
+        "popq %r14\n\t"
+        "popq %r15\n\t"
+        "popq %rbp\n\t"
+        "retq");
+
+__asm__(".global ucontext_trampoline\n"
+        "ucontext_trampoline:\n\t"
+        ".cfi_startproc\n\t"
+        ".cfi_undefined rip\n\t"
+        "movq %r12, %rdi\n\t"
+        "jmpq *%rbx\n\t"
+        ".cfi_endproc");
+
+void ucontext_init(ucontext *context, void *stack, unsigned stack_sz,
+                   void (*func)(void*), void *arg) {
+  void **sp = reinterpret_cast<void **>(static_cast<char *>(stack) + stack_sz);
+  *(--sp) = 0;
+  *(--sp) = reinterpret_cast<void *>(ucontext_trampoline);
+  *(--sp) = 0;   // rbp
+  *(--sp) = 0;   // r15
+  *(--sp) = 0;   // r14
+  *(--sp) = 0;   // r13
+  *(--sp) = arg; // r12
+  *(--sp) = reinterpret_cast<void *>(func); // rbx
+  context->sp = sp;
+  context->fiber = __tsan_create_fiber(0);
+}
+
+void ucontext_free(ucontext *context) {
+  __tsan_destroy_fiber(context->fiber);
+}
+
+__attribute__((no_sanitize_thread))
+void ucontext_switch(ucontext *save, ucontext *load) {
+  save->fiber = __tsan_get_current_fiber();
+  __tsan_switch_to_fiber(load->fiber, 0);
+  ucontext_do_switch(&save->sp, &load->sp);
+}
+
+char stack[64 * 1024] __attribute__((aligned(16)));
+
+ucontext uc, orig_uc;
+
+void func(void *arg) {
+  __asm__ __volatile__(".cfi_undefined rip");
+  ucontext_switch(&uc, &orig_uc);
+}
+
+int main() {
+  ucontext_init(&uc, stack, sizeof(stack), func, 0);
+  ucontext_switch(&orig_uc, &uc);
+  ucontext_free(&uc);
+  fprintf(stderr, "PASS\n");
+  return 0;
+}
+
+// CHECK-NOT: WARNING: ThreadSanitizer:
+// CHECK: PASS

compiler-rt/test/tsan/fiber_from_thread.cc

@@ -0,0 +1,48 @@
+// RUN: %clang_tsan -O1 %s -o %t && %run %t 2>&1 | FileCheck %s
+// UNSUPPORTED: darwin
+#include "test.h"
+#include <ucontext.h>
+
+char stack[64 * 1024] __attribute__((aligned(16)));
+
+ucontext_t uc, orig_uc1, orig_uc2;
+void *fiber, *orig_fiber1, *orig_fiber2;
+
+int var;
+
+void *Thread(void *x) {
+  orig_fiber2 = __tsan_get_current_fiber();
+  swapcontext(&orig_uc2, &orig_uc1);
+  return 0;
+}
+
+void func() {
+  pthread_t t;
+  pthread_create(&t, 0, Thread, 0);
+  pthread_join(t, 0);
+  __tsan_switch_to_fiber(orig_fiber1, 0);
+  swapcontext(&uc, &orig_uc1);
+}
+
+int main() {
+  orig_fiber1 = __tsan_get_current_fiber();
+  fiber = __tsan_create_fiber(0);
+  getcontext(&uc);
+  uc.uc_stack.ss_sp = stack;
+  uc.uc_stack.ss_size = sizeof(stack);
+  uc.uc_link = 0;
+  makecontext(&uc, func, 0);
+  var = 1;
+  __tsan_switch_to_fiber(fiber, 0);
+  swapcontext(&orig_uc1, &uc);
+  var = 2;
+  __tsan_switch_to_fiber(orig_fiber2, 0);
+  swapcontext(&orig_uc1, &orig_uc2);
+  var = 3;
+  __tsan_destroy_fiber(fiber);
+  fprintf(stderr, "PASS\n");
+  return 0;
+}
+
+// CHECK-NOT: WARNING: ThreadSanitizer:
+// CHECK: PASS