/* UPC Runtime Layer API for compiler-generated code * * $Source: bitbucket.org:berkeleylab/upc-runtime.git/upcr.h $ */ /* * This file describes the interface between the platform-independent code * generated by a UPC-to-C translating compiler, and the hand-written UPC * runtime layer that implements the language on a given architecture * * Many/most of the operations below will be implemented using macros or inline * functions in an actual implementation (a number of design decisions in the * interface were based on the expected optimizations that will occur in such an * implementation) They are specified using function declaration syntax below to * make the types clear All correct generated code must type check using the * definitions below In no case should client code assume it can create a * "function pointer" to any of these operations * * Note this interface is meant primarily as a compilation target for a code * generator, not a library for hand-written code - as such, the goals of * expressiveness and performance generally take precedence over readability * and minimality */ #ifndef UPCR_H #define UPCR_H #ifdef __SUNPRO_C /* disable silly warnings which some macros trigger */ #pragma error_messages(off, E_END_OF_LOOP_CODE_NOT_REACHED) #pragma error_messages(off, E_STATEMENT_NOT_REACHED) #pragma error_messages(off, E_LOOP_NOT_ENTERED_AT_TOP) /* disable a warning caused by generated code */ #pragma error_messages(off, E_INC_USR_INC_MAY_NOT_PORTABLE) #endif #ifdef sun /* need this on Solaris to get some commonly-used * functions that users expect (like drand & friends) */ #define __EXTENSIONS__ 1 #endif /* when compiling with gccupc, tell gasnet to use features supported by gccupc regardless of what was probed from the backend C compiler */ #if __GCC_UPC__ || __clang_upc__ /* This upcr supports only gccupc >= 4, but these attributes have all been supported in gccupc since 3.4.4.1 */ #define GASNETT_USE_GCC_ATTRIBUTE_ALWAYSINLINE 1 #define GASNETT_USE_GCC_ATTRIBUTE_NOINLINE 1 #define GASNETT_USE_GCC_ATTRIBUTE_MALLOC 1 #define GASNETT_USE_GCC_ATTRIBUTE_WARNUNUSEDRESULT 1 #define GASNETT_USE_GCC_ATTRIBUTE_USED 1 #define GASNETT_USE_GCC_ATTRIBUTE_UNUSED 1 #define GASNETT_USE_GCC_ATTRIBUTE_MAYALIAS 1 #define GASNETT_USE_GCC_ATTRIBUTE_NORETURN 1 #define GASNETT_USE_GCC_ATTRIBUTE_PURE 1 #define GASNETT_USE_GCC_ATTRIBUTE_CONST 1 #define GASNETT_USE_GCC_ATTRIBUTE_FORMAT 1 #define GASNETT_USE_GCC_ATTRIBUTE_FORMAT_FUNCPTR 1 #define GASNETT_USE_GCC_ATTRIBUTE_FORMAT_FUNCPTR_ARG 1 /* __restrict__ has always been available and usable on typedefs */ #define GASNETT_USE_RESTRICT __restrict__ #define GASNETT_USE_RESTRICT_ON_TYPEDEFS 1 #endif #if defined(__UPC__) /* Bug 2914 - don't define __UPC__ while including system headers */ #undef __UPC__ #define UPCR_DEFINE_UPC #endif #if 1 // TODO-EX: remove this #include #else #include #endif #include #include #ifdef UPCR_DEFINE_UPC #define __UPC__ 1 #endif #if UPCRI_USING_GCCUPC || UPCRI_USING_CUPC #define UPCRI_LIBWRAP 1 #endif #if UPCRI_GASP #include #include #endif #if GASNET_PSHM && !defined(UPCRI_USING_PSHM) /* USING means we perform PSHM-based optimizations within UPCR */ #define UPCRI_USING_PSHM 1 #endif /* -------------------------------------------------------------------------- */ /* * System parameters * ================= * Provided by the runtime system implementation to describe the runtime * environment. Most of this information is probably also made available to * the UPC translator at UPC-to-C compile-time (by some mechanism not * specified here), but some compilers may simply wish to generate generic * code that compiles to have the correct behavior at C compile time using * these preprocessor symbols. */ #define UPCR_MAXNODES GASNET_MAXNODES /* UPCR_PLATFORM_ENVIRONMENT provides the platform-independent UPC compiler with * some clues about the memory layout of the current platform to aid optimization * trade-offs. * The possible configuration values are: * UPCR_PURE_SHARED - purely shared memory, remote memory accesses are handled * entirely by hardware with no software interpretation overhead * UPCR_PURE_DISTRIBUTED - purely distributed memory, remote memory accesses are * handled by some software networking layer * UPCR_SHARED_DISTRIBUTED - a mixture of the above - some remote memory * accesses are handled by hardware, others by a software networking layer * UPCR_OTHER - any configuration not captured by the above options */ #define UPCR_PURE_SHARED 1 #define UPCR_SHARED_DISTRIBUTED 2 #define UPCR_PURE_DISTRIBUTED 3 #if GASNET_CONDUIT_SMP #define UPCR_PLATFORM_ENVIRONMENT UPCR_PURE_SHARED #define UPCRI_PLATENV_STR "shared" #elif (UPCRI_UPC_PTHREADS || GASNET_PSHM) #define UPCR_PLATFORM_ENVIRONMENT UPCR_SHARED_DISTRIBUTED #define UPCRI_PLATENV_STR "shared-distributed" #else #define UPCR_PLATFORM_ENVIRONMENT UPCR_PURE_DISTRIBUTED #define UPCRI_PLATENV_STR "distributed" #endif /* Iff a single shared memory segment used, and it starts at * the same virtual address on all threads */ #if GASNET_ALIGNED_SEGMENTS && !defined(GASNET_SEGMENT_EVERYTHING) && \ !(UPCRI_UPC_PTHREADS || GASNET_PSHM) && 0 /* disabled */ /* NOTE: * In GASNet-EX, a non-zero definition of GASNET_ALIGNED_SEGMENTS is only a * promise that gasnet_attach() will provide alignment, and NOT any guarantee * regarding gex_Segment_Attach (which we use). * So, we cannot rely upon it here. * * TODO-EX: restore/revise the SINGLE_ALIGNED_REGIONS logic using * offset-based addressing once supported in GASNet-EX. */ # define UPCRI_SINGLE_ALIGNED_REGIONS 1 #else # define UPCRI_SINGLE_ALIGNED_REGIONS 0 #endif /* GCCUPC support: */ #if UPCRI_USING_GCCUPC #if UPCRI_STRUCT_SPTR #define UPCR_USING_LINKADDRS 1 #endif #if UPCRI_UPC_PTHREADS #define UPCR_PTHREADS_SECTION #endif #endif /* Clang UPC support: */ #if UPCRI_USING_CUPC #if UPCRI_STRUCT_SPTR #define UPCR_USING_LINKADDRS 1 #endif #endif #if UPCR_USING_LINKADDRS || UPCRI_USING_GCCUPC || UPCRI_USING_CUPC #define UPCRI_HAVE_LINKER_SECTION 1 #endif /* * Translate GASNET macros into UPCR equivalents * only as required by public spec */ #define UPCR_RESTRICT GASNETT_RESTRICT #define UPCR_PAGESIZE GASNETT_PAGESIZE /* use instead of getenv(), which may not get same value on all nodes */ #define upcr_getenv(env_name) gasnet_getenv((env_name)) extern char * bupc_getenv(const char *env_name); /* * Symmetric segments, available on systems that divide the global heap into * equal segments. Happens always on SMP w/o PSHM. */ #if PLATFORM_ARCH_64 && (GASNET_CONDUIT_SMP && UPCRI_UPC_PTHREADS && !GASNET_PSHM) #define UPCRI_SYMMETRIC_SEGMENTS 1 #endif /* User may explicitly set UPCR_DEBUG or UPCR_NDEBUG, otherwise use GASNet's debug setting by default */ #if !defined(UPCR_DEBUG) && !defined(UPCR_NDEBUG) #if defined(GASNET_DEBUG) #define UPCR_DEBUG 1 #elif defined(GASNET_NDEBUG) #define UPCR_NDEBUG 1 #endif #endif #if defined(UPCR_DEBUG) && !defined(UPCR_NDEBUG) #undef UPCR_DEBUG #define UPCR_DEBUG 1 #elif !defined(UPCR_DEBUG) && defined(UPCR_NDEBUG) #undef UPCR_NDEBUG #define UPCR_NDEBUG 1 #else #error bad defns of UPCR_DEBUG and UPCR_NDEBUG #endif #if UPCR_NDEBUG #define upcri_assert(expr) ((void)0) #else #define upcri_assert(expr) \ (GASNETT_PREDICT_TRUE(expr) ? (void)0 : \ upcri_err("Assertion failure at %s: %s", \ gasnett_current_loc, #expr)) #endif #if UPCRI_BUILDING_LIBUPCR || __GCC_UPC__ || __clang_upc__ #define UPCR_NO_SRCPOS 1 #endif #define UPCRI_DEPRECATED_STUB(_fn) \ GASNETT_INLINE(_fn) GASNETT_DEPRECATED void _fn(void) {/*empty*/} /* error handling routines */ #include /* node/thread/pthread logic */ #include /* GASP profiling support */ #include /* utility routines */ #include /* abstract shared pointer interface */ #include /* locks */ #include /* sem */ #include /* barriers */ #include /* collectives */ #include /* I/O */ #include /* Memory allocation functions */ #include /* Atomic extension functions */ #include /* Active Message handlers */ #include /* Shared memory access functions */ #include /* Memcpy extension functions */ #include /* Runtime initialization functions */ #include /* extern functions (use for bootstrapping) */ #include #if __GCC_UPC__ || __clang_upc__ #include #endif /* * Network polling * =============== * * The upcr_poll() function explicitly causes the runtime to attempt to make * progress on any network requests that may be pending. While many other * runtime functions implicitly do this as well (i.e. most of those which call * the network layer) this function may be useful in cases where a large amount * of time has elapsed since the last runtime call (e.g. if a great deal of * application-level calculation is taking place). This function may also be * indirectly when a upc_fence is used. * * upcr_poll() also provides a null strict reference, corresponding to upc_fence in the * UPC memory model. * DOB: we should really rename upcr_poll to upcr_fence, but this would break * compatibility between old runtimes and new translators, so until the next * major runtime interface upgrade, (b)upc_poll expands to upcr_poll_nofence, * which polls without the overhead of strict memory fences. */ /* Bug 2996 - upcr_poll_nofence should also yield in polite mode to get * resonable performance from a spin-loop constructed according to our * recommendations. * The bug was first seen w/ smp-conduit, but when using a network we * cannot claim to know if gasnet_AMPoll() is going to yield or not. * With an RMDA-capable transport one actually could expect that it * would NOT. */ #define upcr_poll_nofence() do { \ gasnet_AMPoll(); \ if_pf (upcri_polite_wait) \ gasnett_sched_yield(); \ } while (0) #if GASNET_CONDUIT_SMP && !UPCRI_UPC_PTHREADS && !GASNET_PSHM /* in the special case of exactly one UPC thread, nothing is required for * correctness of fence (poll is likely a no-op as well, included solely * for tracing purposes) */ #define upcr_poll() upcr_poll_nofence() #else /* in all other cases, a fence needs to act as a null strict reference, * which means we need an architectural membar & optimization barrier to * ensure that surrounding relaxed shared and local operations are not * reordered in any way across this point (which could be visible if other * CPU's or an RDMA enabled NIC are modifying memory via strict operations). * We need both an WMB and RMB within the fence, but it doesn't actually matter * whether they come before or after the optional poll (which is added as * a performance optimization, to help ensure progress in spin-loops using fence). * We combine them in a call to gasnett_local_mb(), which on some architectures * can be slightly more efficient than WMB and RMB called in sequence. */ #define upcr_poll() do { \ upcri_strict_hook(); \ gasnett_local_mb(); \ upcr_poll_nofence(); \ } while (0) #endif /* UPCR_CONFIG_STRING a string representing all the relevant upcr configuration settings that can be compared using string compare to verify version compatibility. The string is also embedded into the library itself such that it can be scanned for within a binary executable. */ #if UPCR_DEBUG #define UPCRI_DEBUG_CONFIG "debug" #else #define UPCRI_DEBUG_CONFIG "nodebug" #endif #if (UPCRI_USING_GCCUPC || UPCRI_USING_CUPC || UPCRI_USING_CUPC2C) && UPCRI_SPTR_ADDR_FIRST #define UPCRI_SPTR_CONFIG_BITS "/a" _STRINGIFY(UPCRI_ADDR_BITS) \ ":t" _STRINGIFY(UPCRI_THREAD_BITS) \ ":p" _STRINGIFY(UPCRI_PHASE_BITS) #else #define UPCRI_SPTR_CONFIG_BITS "/p" _STRINGIFY(UPCRI_PHASE_BITS) \ ":t" _STRINGIFY(UPCRI_THREAD_BITS) \ ":a" _STRINGIFY(UPCRI_ADDR_BITS) #endif #if UPCRI_PACKED_SPTR #define UPCRI_SPTR_CONFIG_BASE "packed" UPCRI_SPTR_CONFIG_BITS #else #define UPCRI_SPTR_CONFIG_BASE "struct" UPCRI_SPTR_CONFIG_BITS #endif #if UPCRI_SYMMETRIC_PSHARED #if __BERKELEY_UPC_POW2_SYMPTR__ #define UPCRI_SPTR_CONFIG "fsymmetric/" UPCRI_SPTR_CONFIG_BASE #else #define UPCRI_SPTR_CONFIG "symmetric/" UPCRI_SPTR_CONFIG_BASE #endif #else #define UPCRI_SPTR_CONFIG UPCRI_SPTR_CONFIG_BASE #endif #if UPCRI_UPC_PTHREADS && GASNET_PSHM #define UPCRI_SHMEM "pthreads/pshm" #elif UPCRI_UPC_PTHREADS #define UPCRI_SHMEM "pthreads" #elif GASNET_PSHM #define UPCRI_SHMEM "pshm" #else #define UPCRI_SHMEM "none" #endif #if UPCRI_GASP #define UPCRI_GASP_CONFIG "gasp" #else #define UPCRI_GASP_CONFIG "nogasp" #endif #if UPCRI_USING_GCCUPC #define UPCRI_TRANS_CONFIG "gupc" #elif UPCRI_USING_CUPC #define UPCRI_TRANS_CONFIG "clangupc" #elif UPCRI_USING_CUPC2C #define UPCRI_TRANS_CONFIG "clangupc2c" #else #define UPCRI_TRANS_CONFIG "berkeleyupc" #endif #define UPCR_CONFIG_STRING "VERSION=" UPCR_VERSION "," \ "PLATFORMENV=" UPCRI_PLATENV_STR "," \ "SHMEM=" UPCRI_SHMEM "," \ "SHAREDPTRREP=" UPCRI_SPTR_CONFIG "," \ "TRANS=" UPCRI_TRANS_CONFIG "," \ UPCRI_DEBUG_CONFIG "," \ UPCRI_GASP_CONFIG /* DOB: would be nice to include UPCR_MAX_BLOCKSIZE, but there's no convenient way to get it as an integer literal at library preprocess time, so punt for now... "MAXBLOCKSIZE=" _STRINGIFY(UPCR_MAX_BLOCKSIZE) "," \ PHH: Still not encoding MAXBLOCKSIZE, but UPCR_MAX_BLOCKSIZE == (1 << UPCRI_PHASE_BITS) and we do now encode UPCRI_PHASE_BITS into SHAREDPTRREP portion of string. */ /* used by upcc when compiling client code */ #if __UPC_STATIC_THREADS__ #ifndef THREADS #error THREADS not defined! #endif #define UPCRI_THREADCONFIG_STR ",staticthreads=" _STRINGIFY(THREADS) #elif __UPC_DYNAMIC_THREADS__ #define UPCRI_THREADCONFIG_STR ",dynamicthreads" #endif #if UPCRI_LIBWRAP #define UPCRI_LIBWRAP_FN __attribute__((__always_inline__)) static __inline__ #ifdef __BERKELEY_UPC_ONLY_PREPROCESS__ /* only during app compile, not libupcr compile */ #include #ifndef BUPC_DISABLE_EXTENSIONS #include #endif #endif #endif /* reverse name shifting done by detect-upc for UPC keywords in C mode files */ #if defined(__BERKELEY_UPC_SECOND_PREPROCESS__) #define _bupc_shared shared #define _bupc_strict strict #define _bupc_relaxed relaxed /* Can't unshift MYTHREAD or THREADS because they are still macros */ #endif /* undef some particularly naughty/dangerous internal functions that should never be available in code outside UPCR */ #if !UPCRI_BUILDING_LIBUPCR #undef upcri_s_islocal #undef upcri_p_islocal #undef upcri_s_nodeof #undef upcri_p_nodeof #undef upcri_s2local #undef upcri_s2localoff #undef upcri_p2local #undef upcri_p2localoff #define _upcri_shared_to_remote abort #define _upcri_shared_to_remote_off abort #define _upcri_shared_to_remote_withthread abort #define _upcri_pshared_to_remote abort #define _upcri_pshared_to_remote_off abort #define _upcri_pshared_to_remote_withthread abort #define _upcri_shared_remote_to_mylocal abort #endif /* In backend compilation stage we provide the following UPC library(-like) * functions with a "upc_" prefix in addition to the "upcr_" prefixed * versions in the runtime specification (see bug 1109). As of the 3.6 * revision of the runtime spec, a translator is no longer required to * name-shift these idenitifiers (though they are still permitted to). */ #if defined(__BERKELEY_UPC_SECOND_PREPROCESS__) #define upc_global_exit upcr_global_exit #define upc_global_alloc upcr_global_alloc #define upc_all_alloc upcr_all_alloc #define upc_alloc upcr_alloc #define upc_local_alloc upcr_local_alloc #define upc_free upcr_free #define upc_all_free upcr_all_free #define upc_threadof upcr_threadof_shared #define upc_phaseof upcr_phaseof_shared #define upc_addrfield upcr_addrfield_shared #define upc_affinitysize upcr_affinitysize #define upc_resetphase upcr_shared_resetphase #define upc_global_lock_alloc upcr_global_lock_alloc #define upc_all_lock_alloc upcr_all_lock_alloc #define upc_lock_free upcr_lock_free #define upc_all_lock_free upcr_all_lock_free #define upc_lock upcr_lock #define upc_lock_attempt upcr_lock_attempt #define upc_unlock upcr_unlock #define upc_memcpy upcr_memcpy #define upc_memput upcr_memput #define upc_memget upcr_memget #define upc_memset upcr_memset #define upc_cast upcr_cast #define upc_thread_info upcr_thread_info #endif /* __BERKELEY_UPC_SECOND_PREPROCESS__ */ #if defined(__BERKELEY_UPC_SECOND_PREPROCESS__) /* The following do NOT corespond to any current UPC specification. Nor are they part of the current runtime specification. However, this is probably the best place for these definitions. Note that gupc gets definitions in upcr_preinclude/bupc_extensions.h */ #define bupc_poll() upcr_poll_nofence() #define upc_poll() upcr_poll_nofence() /* for backward-compatibility */ /* The following is not (yet) part of the runtime specification. However, it is already in use by the both the Berkeley and clang-upc2c translators; and it is under consideration for inclusion in a future UPCR spec revision. */ #define upcr_barrier(x,y) do {\ int _x = (x), _y = (y); \ upcr_notify(_x,_y); \ upcr_wait(_x,_y); \ } while (0) /* The following mapping of upc_nb.h is not yet in the UPCR spec: */ #define upc_memcpy_nb upcr_memcpy_nb #define upc_memget_nb upcr_memget_nb #define upc_memput_nb upcr_memput_nb #define upc_memset_nb upcr_memset_nb #define upc_memget_nbi upcr_nbi_memget #define upc_memput_nbi upcr_nbi_memput #define upc_memcpy_nbi upcr_nbi_memcpy #define upc_memset_nbi upcr_nbi_memset #define upc_sync upcr_waitsync #define upc_sync_attempt upcr_trysync #define upc_synci upcr_wait_syncnbi_all #define upc_synci_attempt upcr_try_syncnbi_all #endif /* __BERKELEY_UPC_SECOND_PREPROCESS__ */ #endif /* UPCR_H */