Add AO primitives implementation to GC based on C11 atomic intrinsic

* Makefile.direct (SRCS): Add include/private/gc_atomic_ops.h.
* include/include.am (dist_noinst_HEADERS): Likewise.
* include/private/gc_atomic_ops.h: New file; define needed AO entities
if GC_BUILTIN_ATOMIC, otherwise include atomic_ops.h.
* include/private/gc_locks.h [THREADS && GC_PTHREADS
&& !GC_WIN32_THREADS]: Include gc_atomic_ops.h instead of atomic_ops.h.
* include/private/gc_priv.h [PARALLEL_MARK || THREADS && MPROTECT_VDB]:
Likewise.
* include/private/specific.h: Likewise.
* pthread_stop_world.c: Include private/gc_atomic_ops.h instead of
atomic_ops.h.
* tests/subthread_create.c: Likewise.
* thread_local_alloc.c [THREAD_LOCAL_ALLOC && GC_GCJ_SUPPORT]: Likewise.
* typd_mlc.c [GC_FORCE_INCLUDE_ATOMIC_OPS || GC_BUILTIN_ATOMIC]:
Likewise.
* win32_threads.c [GC_DLL && !GC_NO_THREADS_DISCOVERY && !MSWINCE]:
Likewise.

Makefile.direct

@@ -111,7 +111,7 @@ SRCS= $(CSRCS) \
   include/javaxfc.h \
   include/gc_backptr.h include/gc_gcj.h include/private/dbg_mlc.h \
   include/private/specific.h include/leak_detector.h \
-  include/gc_pthread_redirects.h \
+  include/gc_pthread_redirects.h include/private/gc_atomic_ops.h \
   include/gc_config_macros.h include/private/pthread_support.h \
   include/private/pthread_stop_world.h include/private/darwin_semaphore.h \
   include/private/darwin_stop_world.h include/private/thread_local_alloc.h \

include/include.am

@@ -39,6 +39,7 @@ dist_noinst_HEADERS += \
         include/private/darwin_semaphore.h \
         include/private/darwin_stop_world.h \
         include/private/dbg_mlc.h \
+        include/private/gc_atomic_ops.h \
         include/private/gc_hdrs.h \
         include/private/gc_locks.h \
         include/private/gc_pmark.h \

include/private/gc_atomic_ops.h

@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2017 Ivan Maidanski
+ *
+ * THIS MATERIAL IS PROVIDED AS IS, WITH ABSOLUTELY NO WARRANTY EXPRESSED
+ * OR IMPLIED.  ANY USE IS AT YOUR OWN RISK.
+ *
+ * Permission is hereby granted to use or copy this program
+ * for any purpose,  provided the above notices are retained on all copies.
+ * Permission to modify the code and to distribute modified code is granted,
+ * provided the above notices are retained, and a notice that the code was
+ * modified is included with the above copyright notice.
+ */
+
+/* This is a private GC header which provides an implementation of      */
+/* libatomic_ops subset primitives sufficient for GC assuming that C11  */
+/* atomic intrinsics are available (and have correct implementation).   */
+/* This is enabled by defining GC_BUILTIN_ATOMIC macro.  Otherwise,     */
+/* libatomic_ops library is used to define the primitives.              */
+
+#ifndef GC_ATOMIC_OPS_H
+#define GC_ATOMIC_OPS_H
+
+#ifdef GC_BUILTIN_ATOMIC
+
+# include "gc.h" /* for GC_word */
+  typedef GC_word AO_t;
+
+# ifdef GC_INLINE
+#   define AO_INLINE GC_INLINE
+# else
+#   define AO_INLINE static __inline
+# endif
+
+  typedef unsigned char AO_TS_t;
+# define AO_TS_CLEAR 0
+# define AO_TS_INITIALIZER (AO_TS_t)AO_TS_CLEAR
+# ifdef __GCC_ATOMIC_TEST_AND_SET_TRUEVAL
+#   define AO_TS_SET __GCC_ATOMIC_TEST_AND_SET_TRUEVAL
+# else
+#   define AO_TS_SET (AO_TS_t)0xff
+# endif
+# define AO_CLEAR(p) __atomic_clear(p, __ATOMIC_RELEASE)
+# define AO_test_and_set_acquire(p) __atomic_test_and_set(p, __ATOMIC_ACQUIRE)
+# define AO_HAVE_test_and_set_acquire
+
+# define AO_compiler_barrier() __atomic_thread_fence(__ATOMIC_RELAXED)
+# define AO_nop_full() __atomic_thread_fence(__ATOMIC_SEQ_CST)
+# define AO_HAVE_nop_full
+
+# define AO_fetch_and_add(p, v) __atomic_fetch_add(p, v, __ATOMIC_RELAXED)
+# define AO_HAVE_fetch_and_add
+# define AO_fetch_and_add1(p) AO_fetch_and_add(p, 1)
+# define AO_HAVE_fetch_and_add1
+
+# define AO_or(p, v) (void)__atomic_or_fetch(p, v, __ATOMIC_RELAXED)
+# define AO_HAVE_or
+
+# define AO_load(p) __atomic_load_n(p, __ATOMIC_RELAXED)
+# define AO_HAVE_load
+# define AO_load_acquire(p) __atomic_load_n(p, __ATOMIC_ACQUIRE)
+# define AO_HAVE_load_acquire
+# define AO_load_acquire_read(p) AO_load_acquire(p)
+# define AO_HAVE_load_acquire_read
+
+# define AO_store(p, v) __atomic_store_n(p, v, __ATOMIC_RELAXED)
+# define AO_HAVE_store
+# define AO_store_release(p, v) __atomic_store_n(p, v, __ATOMIC_RELEASE)
+# define AO_HAVE_store_release
+# define AO_store_release_write(p, v) AO_store_release(p, v)
+# define AO_HAVE_store_release_write
+
+# ifdef AO_REQUIRE_CAS
+    AO_INLINE int
+    AO_compare_and_swap(volatile AO_t *p, AO_t ov, AO_t nv)
+    {
+      return (int)__atomic_compare_exchange_n(p, &ov, nv, 0,
+                                        __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+    }
+#   define AO_HAVE_compare_and_swap
+
+    AO_INLINE int
+    AO_compare_and_swap_release(volatile AO_t *p, AO_t ov, AO_t nv)
+    {
+      return (int)__atomic_compare_exchange_n(p, &ov, nv, 0,
+                                        __ATOMIC_RELEASE, __ATOMIC_RELAXED);
+    }
+#   define AO_HAVE_compare_and_swap_release
+# endif
+
+#else
+  /* Fallback to libatomic_ops. */
+# include "atomic_ops.h"
+#endif /* !GC_BUILTIN_ATOMIC */
+
+#endif /* GC_ATOMIC_OPS_H */

include/private/gc_locks.h

@@ -29,7 +29,7 @@
 # ifdef THREADS
 
 #  if defined(GC_PTHREADS) && !defined(GC_WIN32_THREADS)
-#    include "atomic_ops.h"
+#    include "gc_atomic_ops.h"
 #  endif
 
 #  ifdef PCR

include/private/gc_priv.h

@@ -927,13 +927,13 @@ typedef word page_hash_table[PHT_SIZE];
            /* to allocate smaller header for large objects.  */
 
 #ifdef PARALLEL_MARK
-# include "atomic_ops.h"
+# include "gc_atomic_ops.h"
 # define counter_t volatile AO_t
 #else
   typedef size_t counter_t;
 # if defined(THREADS) && (defined(MPROTECT_VDB) \
                 || (defined(GC_ASSERTIONS) && defined(THREAD_LOCAL_ALLOC)))
-#   include "atomic_ops.h"
+#   include "gc_atomic_ops.h"
 # endif
 #endif /* !PARALLEL_MARK */
 

include/private/specific.h

@@ -13,7 +13,8 @@
  */
 
 #include <errno.h>
-#include "atomic_ops.h"
+
+#include "gc_atomic_ops.h"
 
 /* Called during key creation or setspecific.           */
 /* For the GC we already hold lock.                     */

pthread_stop_world.c

@@ -45,7 +45,8 @@
 #include <semaphore.h>
 #include <errno.h>
 #include <unistd.h>
-#include "atomic_ops.h"
+
+#include "private/gc_atomic_ops.h"
 
 /* It's safe to call original pthread_sigmask() here.   */
 #undef pthread_sigmask

tests/subthread_create.c

@@ -12,7 +12,7 @@
 #ifdef PARALLEL_MARK
 # define AO_REQUIRE_CAS
 #endif
-#include "atomic_ops.h"
+#include "private/gc_atomic_ops.h"
 
 #include <stdio.h>
 

thread_local_alloc.c

@@ -193,7 +193,7 @@ GC_API GC_ATTR_MALLOC void * GC_CALL GC_malloc_kind(size_t bytes, int knd)
 
 #ifdef GC_GCJ_SUPPORT
 
-# include "atomic_ops.h" /* for AO_compiler_barrier() */
+# include "private/gc_atomic_ops.h" /* for AO_compiler_barrier() */
 
 # include "include/gc_gcj.h"
 

typd_mlc.c

@@ -99,8 +99,9 @@ STATIC size_t GC_avail_descr = 0;       /* Next available slot.         */
 STATIC int GC_typed_mark_proc_index = 0; /* Indices of my mark          */
 STATIC int GC_array_mark_proc_index = 0; /* procedures.                 */
 
-#if !defined(AO_HAVE_load_acquire) && defined(GC_FORCE_INCLUDE_ATOMIC_OPS)
-# include "atomic_ops.h"
+#if !defined(AO_HAVE_load_acquire) \
+    && (defined(GC_FORCE_INCLUDE_ATOMIC_OPS) || defined(GC_BUILTIN_ATOMIC))
+# include "private/gc_atomic_ops.h"
 #endif
 
 STATIC

win32_threads.c

@@ -79,7 +79,7 @@
 #if (defined(GC_DLL) || defined(GC_INSIDE_DLL)) \
         && !defined(GC_NO_THREADS_DISCOVERY) && !defined(MSWINCE) \
         && !defined(THREAD_LOCAL_ALLOC) && !defined(GC_PTHREADS)
-# include "atomic_ops.h"
+# include "private/gc_atomic_ops.h"
 
   /* This code operates in two distinct modes, depending on     */
   /* the setting of GC_win32_dll_threads.                       */