test.cc

//
// This is a reproduction for the race between forking and libnotify
// intialization.
//
// # check that there are no core dumps
// $ ls /cores
// # set core limit
// $ ulimit -c unlimited
// # build the test file
// $ clang++ -o test test.cc
// # run the test. note: must run a lot concurrently
// # because it's a race!
// $ for i in $(seq 0 1000); do; ./test & ; done;
// # now there will be core files
// $ ls /cores
// core.24377 core.24520
// $ lldb test -c /cores/core.24377
// (lldb) target create "test" --core "/cores/core.24377"
// warning: (x86_64) /cores/core.24377 load command 374 LC_SEGMENT_64 has a fileoff + filesize (0x2cf57000) that extends beyond the end of the file (0x2cf56000), the segment will be truncated to match
// warning: (x86_64) /cores/core.24377 load command 375 LC_SEGMENT_64 has a fileoff (0x2cf57000) that extends beyond the end of the file (0x2cf56000), ignoring this section
// Core file '/cores/core.24377' (x86_64) was loaded.
// (lldb) bt
// * thread #1: tid = 0x0000, 0x00007fffbea85c87 libsystem_platform.dylib`_os_once_gate_corruption_abort + 23, stop reason = signal SIGSTOP
//   * frame #0: 0x00007fffbea85c87 libsystem_platform.dylib`_os_once_gate_corruption_abort + 23
//     frame #1: 0x00007fffbea8599b libsystem_platform.dylib`_os_once_gate_wait_slow + 134
//     frame #2: 0x00007fffbea81a92 libsystem_platform.dylib`_os_alloc_once + 40
//     frame #3: 0x00007fffbea7b0ae libsystem_notify.dylib`_notify_fork_child + 215
//     frame #4: 0x00007fffbd2ccb21 libSystem.B.dylib`libSystem_atfork_child + 49
//     frame #5: 0x00007fffbe8b9437 libsystem_c.dylib`fork + 47
//     frame #6: 0x000000010c3c8956 test`ForkThread(void*) + 150
//     frame #7: 0x00007fffbea8c9af libsystem_pthread.dylib`_pthread_body + 180
//     frame #8: 0x00007fffbea8c8fb libsystem_pthread.dylib`_pthread_start + 286
//     frame #9: 0x00007fffbea8c101 libsystem_pthread.dylib`thread_start + 13
// (lldb)
//

#include <cassert>
#include <cstdio>
#include <pthread.h>
#include <unistd.h>
#include <time.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/wait.h>

typedef void* (*ThreadMain)(void*);

// We are peaking into internal libsystem state to reproduce the
// race more reliably.
#define OS_ALLOC_ONCE_KEY_LIBSYSTEM_NOTIFY 0
struct _os_alloc_once_s {
  intptr_t once;
  void* ptr;
};
extern "C" _os_alloc_once_s _os_alloc_once_table[];

struct ThreadState {
  enum State {
    kStarting = 0,
    kReady = 1,
    kDone = 2
  };

  volatile int32_t* me;
  volatile int32_t* other;

  void WaitOther() {
    int32_t v;
    do {
      __atomic_load(other, &v, __ATOMIC_SEQ_CST);
    } while (v == kStarting);
  }

  void SetMe(State state) {
    int32_t v = state;
    __atomic_store(me, &v, __ATOMIC_SEQ_CST);
  }
};


// This thread calls localtime_r which will call into libnotify and
// _notify_globals() will be called to create global state for libnotify.
// While the other thread will try to fork.
void* LibNotifyInitThread(void* data) {
  ThreadState* state = (ThreadState*) data;

  struct timeval tv;
  const bool ok = gettimeofday(&tv, NULL) >= 0;
  assert(ok);

  // Signal that we are ready to race and wait for another thread - which is
  // doing forking - to be up and running.
  state->SetMe(ThreadState::kReady);
  state->WaitOther();

  // This should print NULL because libnotify state is not yet created.
  printf("[init ] _os_alloc_once_table[OS_ALLOC_ONCE_KEY_LIBSYSTEM_NOTIFY] = %lx\n", _os_alloc_once_table[OS_ALLOC_ONCE_KEY_LIBSYSTEM_NOTIFY].once);

  // Call localtime_r which would call into libnotify (probably to subscribe
  // to timezone changes), which would trigget _notify_globals() call.
  // And as a result this thread will start creating libnotify state.
  {
    tm decomposed;
    struct tm* res = localtime_r(&(tv.tv_sec), &decomposed);
    assert(res != NULL);
  }

  // We are done here.
  printf("[init ] done\n");
  state->SetMe(ThreadState::kDone);
  return NULL;
}


// This thread is attempting to race libnotify initialization by triggering
// fork right after _os_alloc_once_table[OS_ALLOC_ONCE_KEY_LIBSYSTEM_NOTIFY].once
// stops being zero - which indicates that another thread
// locked _os_alloc_once_table[OS_ALLOC_ONCE_KEY_LIBSYSTEM_NOTIFY] entry and is
// in the process of initializing it.
void* ForkThread(void* data) {
  ThreadState* state = (ThreadState*) data;

  // Ready to race. Signal other thread.
  state->SetMe(ThreadState::kReady);
  state->WaitOther();

  // Wait for _os_alloc_once_table[OS_ALLOC_ONCE_KEY_LIBSYSTEM_NOTIFY].once to
  // stop being 0 - which means it is being initialized.
  // (if it's -1 then entry has been already created and we are too late)
  printf("[fork ] waiting for the race\n");
  {
    intptr_t once;
    while ((once = _os_alloc_once_table[OS_ALLOC_ONCE_KEY_LIBSYSTEM_NOTIFY].once) == 0);
    printf("[fork ] _os_alloc_once_table[OS_ALLOC_ONCE_KEY_LIBSYSTEM_NOTIFY] = %lx\n", once);
    if (once == -1) {
      printf("[fork ] too late for a race\n");
      exit(0);
    }
  }

  // NOW RACE!
  pid_t pid;
  if ((pid = fork()) != 0) {
    // We are in the parent - wait for the child.
    assert(pid != -1);

    int status;
    if ( waitpid(pid, &status, 0) == -1 ) {
      printf("[fork ] waitpid failed\n");
    } else if ( WIFEXITED(status) ) {
      printf("[fork ] child exited with %d\n", WEXITSTATUS(status));
    }

    printf("[fork ] done\n");
    state->SetMe(ThreadState::kDone);
  } else {
    // We are in the child. Just exit.
    printf("[child] _os_alloc_once_table[OS_ALLOC_ONCE_KEY_LIBSYSTEM_NOTIFY] = %lx\n", _os_alloc_once_table[OS_ALLOC_ONCE_KEY_LIBSYSTEM_NOTIFY].once);
    printf("[child] done\n");
    exit(0);
  }
  return NULL;
}

pthread_t Start(ThreadMain f, ThreadState* state) {
  int res;

  pthread_attr_t attr;
  res = pthread_attr_init(&attr);
  assert(!res);
  res = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
  assert(!res);

  pthread_t id;
  res = pthread_create(&id, &attr, f, state);
  assert(!res);

  res = pthread_attr_destroy(&attr);
  assert(!res);

  return id;
}

int main(int argc, char* argv[]) {
  volatile int32_t state1 = ThreadState::kStarting;
  volatile int32_t state2 = ThreadState::kStarting;

  ThreadState thread1_state = { &state1, &state2 };
  ThreadState thread2_state = { &state2, &state1 };

  // Start two threads (one forking, one touching libnotify).
  Start(&LibNotifyInitThread, &thread1_state);
  Start(&ForkThread, &thread2_state);

  // Wait upto 10 seconds for these threads to complete.
  // If they don't this usually means that we are stuck in waitpid
  // because fork child has died.
  int32_t res1 = 0, res2 = 0;
  int n = 0;
  do {
    if (n++ > 10) {
      printf("[main] something is wrong (check /cores/)\n");
      exit(1);
    }
    sleep(1);
    __atomic_load(&state1, &res1, __ATOMIC_SEQ_CST);
    __atomic_load(&state2, &res2, __ATOMIC_SEQ_CST);
  } while (res1 != ThreadState::kDone || res2 != ThreadState::kDone);
}