17 #ifndef __STARPU_UTIL_H__ 18 #define __STARPU_UTIL_H__ 32 #ifdef STARPU_SIMGRID_MC 33 #include <simgrid/modelchecker.h> 52 #if defined __GNUC__ && defined __GNUC_MINOR__ 53 # define STARPU_GNUC_PREREQ(maj, min) \ 54 ((__GNUC__ << 16) + __GNUC_MINOR__ >= ((maj) << 16) + (min)) 56 # define STARPU_GNUC_PREREQ(maj, min) 0 64 # define STARPU_UNLIKELY(expr) (__builtin_expect(!!(expr),0)) 66 # define STARPU_UNLIKELY(expr) (expr) 74 # define STARPU_LIKELY(expr) (__builtin_expect(!!(expr),1)) 76 # define STARPU_LIKELY(expr) (expr) 83 # define STARPU_ATTRIBUTE_UNUSED __attribute__((unused)) 85 # define STARPU_ATTRIBUTE_UNUSED 92 # define STARPU_ATTRIBUTE_NORETURN __attribute__((noreturn)) 94 # define STARPU_ATTRIBUTE_NORETURN 101 # define STARPU_ATTRIBUTE_INTERNAL __attribute__ ((visibility ("internal"))) 103 # define STARPU_ATTRIBUTE_INTERNAL 110 # define STARPU_ATTRIBUTE_MALLOC __attribute__((malloc)) 112 # define STARPU_ATTRIBUTE_MALLOC 119 # define STARPU_ATTRIBUTE_WARN_UNUSED_RESULT __attribute__((warn_unused_result)) 121 # define STARPU_ATTRIBUTE_WARN_UNUSED_RESULT 128 # define STARPU_ATTRIBUTE_PURE __attribute__((pure)) 130 # define STARPU_ATTRIBUTE_PURE 137 # define STARPU_ATTRIBUTE_ALIGNED(size) __attribute__((aligned(size))) 139 # define STARPU_ATTRIBUTE_ALIGNED(size) 143 # define STARPU_ATTRIBUTE_FORMAT(type, string, first) __attribute__((format(type, string, first))) 145 # define STARPU_ATTRIBUTE_FORMAT(type, string, first) 150 #if defined(c_plusplus) || defined(__cplusplus) 151 # define STARPU_INLINE inline 152 #elif defined(_MSC_VER) || defined(__HP_cc) 153 # define STARPU_INLINE __inline 155 # define STARPU_INLINE __inline__ 158 #if STARPU_GNUC_PREREQ(4, 3) 159 # define STARPU_ATTRIBUTE_CALLOC_SIZE(num,size) __attribute__((alloc_size(num,size))) 160 # define STARPU_ATTRIBUTE_ALLOC_SIZE(size) __attribute__((alloc_size(size))) 162 # define STARPU_ATTRIBUTE_CALLOC_SIZE(num,size) 163 # define STARPU_ATTRIBUTE_ALLOC_SIZE(size) 166 #if STARPU_GNUC_PREREQ(3, 1) && !defined(BUILDING_STARPU) && !defined(STARPU_USE_DEPRECATED_API) && !defined(STARPU_USE_DEPRECATED_ONE_ZERO_API) 167 #define STARPU_DEPRECATED __attribute__((__deprecated__)) 169 #define STARPU_DEPRECATED 172 #if STARPU_GNUC_PREREQ(3,3) 173 #define STARPU_WARN_UNUSED_RESULT __attribute__((__warn_unused_result__)) 175 #define STARPU_WARN_UNUSED_RESULT 178 #define STARPU_BACKTRACE_LENGTH 32 180 # define STARPU_DUMP_BACKTRACE() do { \ 181 void *__ptrs[STARPU_BACKTRACE_LENGTH]; \ 182 int __n = backtrace(__ptrs, STARPU_BACKTRACE_LENGTH); \ 183 backtrace_symbols_fd(__ptrs, __n, 2); \ 186 # define STARPU_DUMP_BACKTRACE() do { } while (0) 189 #ifdef STARPU_SIMGRID_MC 190 #define STARPU_SIMGRID_ASSERT(x) MC_assert(!!(x)) 192 #define STARPU_SIMGRID_ASSERT(x) 199 #ifdef STARPU_NO_ASSERT 200 #define STARPU_ASSERT(x) do { if (0) { (void) (x); } } while(0) 202 # if defined(__CUDACC__) || defined(STARPU_HAVE_WINDOWS) 203 # define STARPU_ASSERT(x) do { if (STARPU_UNLIKELY(!(x))) { STARPU_DUMP_BACKTRACE(); STARPU_SIMGRID_ASSERT(x); *(int*)NULL = 0; } } while(0) 205 # define STARPU_ASSERT(x) do { if (STARPU_UNLIKELY(!(x))) { STARPU_DUMP_BACKTRACE(); STARPU_SIMGRID_ASSERT(x); assert(x); } } while (0) 209 #ifdef STARPU_NO_ASSERT 210 #define STARPU_ASSERT_ACCESSIBLE(x) do { if (0) { (void) (x); } } while(0) 212 #define STARPU_ASSERT_ACCESSIBLE(ptr) do { volatile char __c STARPU_ATTRIBUTE_UNUSED = *(char*) (ptr); } while(0) 220 #ifdef STARPU_NO_ASSERT 221 #define STARPU_ASSERT_MSG(x, msg, ...) do { if (0) { (void) (x); (void) msg; } } while(0) 223 # if defined(__CUDACC__) || defined(STARPU_HAVE_WINDOWS) 224 # define STARPU_ASSERT_MSG(x, msg, ...) do { if (STARPU_UNLIKELY(!(x))) { STARPU_DUMP_BACKTRACE(); fprintf(stderr, "\n[starpu][%s][assert failure] " msg "\n\n", __starpu_func__, ## __VA_ARGS__); STARPU_SIMGRID_ASSERT(x); *(int*)NULL = 0; }} while(0) 226 # define STARPU_ASSERT_MSG(x, msg, ...) do { if (STARPU_UNLIKELY(!(x))) { STARPU_DUMP_BACKTRACE(); fprintf(stderr, "\n[starpu][%s][assert failure] " msg "\n\n", __starpu_func__, ## __VA_ARGS__); STARPU_SIMGRID_ASSERT(x); assert(x); } } while(0) 231 # ifdef __clang_analyzer__ 232 # define _starpu_abort() exit(42) 234 # define _starpu_abort() *(volatile int*)NULL = 0 237 # define _starpu_abort() abort() 243 #define STARPU_ABORT() do { \ 244 STARPU_DUMP_BACKTRACE(); \ 245 fprintf(stderr, "[starpu][abort][%s()@%s:%d]\n", __starpu_func__, __FILE__, __LINE__); \ 254 #define STARPU_ABORT_MSG(msg, ...) do { \ 255 STARPU_DUMP_BACKTRACE(); \ 256 fprintf(stderr, "[starpu][abort][%s()@%s:%d] " msg "\n", __starpu_func__, __FILE__, __LINE__, ## __VA_ARGS__); \ 260 #if defined(_MSC_VER) 261 #undef STARPU_HAVE_STRERROR_R 264 #if defined(STARPU_HAVE_STRERROR_R) 265 #if (! defined(__GLIBC__) || !__GLIBC__) || ((_POSIX_C_SOURCE >= 200112L || _XOPEN_SOURCE >= 600) && (! defined(_GNU_SOURCE))) 267 # define starpu_strerror_r(errnum, buf, buflen) \ 270 int _ret = strerror_r((errnum), (buf), (buflen)); \ 271 STARPU_ASSERT(_ret == 0); \ 276 # define starpu_strerror_r(errnum, buf, buflen) \ 279 char * const _user_buf = (buf); \ 280 const size_t _user_buflen = (buflen); \ 282 STARPU_ASSERT((buf) != NULL); \ 283 char * _tmp_buf = strerror_r((errnum), _user_buf, _user_buflen); \ 284 if (_tmp_buf != _user_buf) \ 286 if (_user_buflen > 0) \ 288 strncpy(_user_buf, _tmp_buf, _user_buflen-1); \ 289 _user_buf[_user_buflen-1] = '\0'; \ 301 #if defined(STARPU_HAVE_STRERROR_R) 302 # define STARPU_CHECK_RETURN_VALUE(err, message, ...) {if (STARPU_UNLIKELY(err != 0)) { \ 303 char xmessage[256]; starpu_strerror_r(-err, xmessage, 256); \ 304 fprintf(stderr, "[starpu] Unexpected value: <%d:%s> returned for " message "\n", err, xmessage, ## __VA_ARGS__); \ 307 # define STARPU_CHECK_RETURN_VALUE(err, message, ...) {if (STARPU_UNLIKELY(err != 0)) { \ 308 fprintf(stderr, "[starpu] Unexpected value: <%d> returned for " message "\n", err, ## __VA_ARGS__); \ 316 #if defined(STARPU_HAVE_STRERROR_R) 317 # define STARPU_CHECK_RETURN_VALUE_IS(err, value, message, ...) {if (STARPU_UNLIKELY(err != value)) { \ 318 char xmessage[256]; starpu_strerror_r(-err, xmessage, 256); \ 319 fprintf(stderr, "[starpu] Unexpected value: <%d!=%d:%s> returned for " message "\n", err, value, xmessage, ## __VA_ARGS__); \ 322 # define STARPU_CHECK_RETURN_VALUE_IS(err, value, message, ...) {if (STARPU_UNLIKELY(err != value)) { \ 323 fprintf(stderr, "[starpu] Unexpected value: <%d != %d> returned for " message "\n", err, value, ## __VA_ARGS__); \ 333 #if defined(__i386__) || defined(__x86_64__) 334 static __starpu_inline
unsigned _starpu_cmpxchg(
unsigned *ptr,
unsigned old,
unsigned next)
336 __asm__ __volatile__(
"lock cmpxchgl %2,%1":
"+a" (old),
"+m" (*ptr) :
"q" (next) :
"memory");
339 #define STARPU_HAVE_CMPXCHG 340 static __starpu_inline
unsigned _starpu_xchg(
unsigned *ptr,
unsigned next)
343 __asm__ __volatile__(
"xchgl %1,%0":
"+m" (*ptr),
"+q" (next) : :
"memory");
346 #define STARPU_HAVE_XCHG 348 static __starpu_inline uint32_t _starpu_cmpxchg32(uint32_t *ptr, uint32_t old, uint32_t next)
350 __asm__ __volatile__(
"lock cmpxchgl %2,%1":
"+a" (old),
"+m" (*ptr) :
"q" (next) :
"memory");
353 #define STARPU_HAVE_CMPXCHG32 354 static __starpu_inline uint32_t _starpu_xchg32(uint32_t *ptr, uint32_t next)
357 __asm__ __volatile__(
"xchgl %1,%0":
"+m" (*ptr),
"+q" (next) : :
"memory");
360 #define STARPU_HAVE_XCHG32 362 #if defined(__i386__) 363 static __starpu_inline
unsigned long _starpu_cmpxchgl(
unsigned long *ptr,
unsigned long old,
unsigned long next)
365 __asm__ __volatile__(
"lock cmpxchgl %2,%1":
"+a" (old),
"+m" (*ptr) :
"q" (next) :
"memory");
368 #define STARPU_HAVE_CMPXCHGL 369 static __starpu_inline
unsigned long _starpu_xchgl(
unsigned long *ptr,
unsigned long next)
372 __asm__ __volatile__(
"xchgl %1,%0":
"+m" (*ptr),
"+q" (next) : :
"memory");
375 #define STARPU_HAVE_XCHGL 378 #if defined(__x86_64__) 379 static __starpu_inline
unsigned long _starpu_cmpxchgl(
unsigned long *ptr,
unsigned long old,
unsigned long next)
381 __asm__ __volatile__(
"lock cmpxchgq %2,%1":
"+a" (old),
"+m" (*ptr) :
"q" (next) :
"memory");
384 #define STARPU_HAVE_CMPXCHGL 385 static __starpu_inline
unsigned long _starpu_xchgl(
unsigned long *ptr,
unsigned long next)
388 __asm__ __volatile__(
"xchgq %1,%0":
"+m" (*ptr),
"+q" (next) : :
"memory");
391 #define STARPU_HAVE_XCHGL 394 #if defined(__i386__) 395 static __starpu_inline uint64_t _starpu_cmpxchg64(uint64_t *ptr, uint64_t old, uint64_t next)
397 uint32_t next_hi = next >> 32;
398 uint32_t next_lo = next & 0xfffffffful;
399 __asm__ __volatile__(
"lock cmpxchg8b %1":
"+A" (old),
"+m" (*ptr) :
"c" (next_hi),
"b" (next_lo) :
"memory");
402 #define STARPU_HAVE_CMPXCHG64 405 #if defined(__x86_64__) 406 static __starpu_inline uint64_t _starpu_cmpxchg64(uint64_t *ptr, uint64_t old, uint64_t next)
408 __asm__ __volatile__(
"lock cmpxchgq %2,%1":
"+a" (old),
"+m" (*ptr) :
"q" (next) :
"memory");
411 #define STARPU_HAVE_CMPXCHG64 412 static __starpu_inline uint64_t _starpu_xchg64(uint64_t *ptr, uint64_t next)
415 __asm__ __volatile__(
"xchgq %1,%0":
"+m" (*ptr),
"+q" (next) : :
"memory");
418 #define STARPU_HAVE_XCHG64 423 #define STARPU_ATOMIC_SOMETHING(name,expr) \ 424 static __starpu_inline unsigned starpu_atomic_##name(unsigned *ptr, unsigned value) \ 426 unsigned old, next; \ 431 if (_starpu_cmpxchg(ptr, old, next) == old) \ 436 #define STARPU_ATOMIC_SOMETHINGL(name,expr) \ 437 static __starpu_inline unsigned long starpu_atomic_##name##l(unsigned long *ptr, unsigned long value) \ 439 unsigned long old, next; \ 444 if (_starpu_cmpxchgl(ptr, old, next) == old) \ 449 #define STARPU_ATOMIC_SOMETHING64(name,expr) \ 450 static __starpu_inline uint64_t starpu_atomic_##name##64(uint64_t *ptr, uint64_t value) \ 452 uint64_t old, next; \ 457 if (_starpu_cmpxchg64(ptr, old, next) == old) \ 464 #ifdef STARPU_HAVE_SYNC_FETCH_AND_ADD 465 #define STARPU_ATOMIC_ADD(ptr, value) (__sync_fetch_and_add ((ptr), (value)) + (value)) 466 #define STARPU_ATOMIC_ADDL(ptr, value) (__sync_fetch_and_add ((ptr), (value)) + (value)) 467 #define STARPU_ATOMIC_ADD64(ptr, value) (__sync_fetch_and_add ((ptr), (value)) + (value)) 469 #if defined(STARPU_HAVE_CMPXCHG) 470 STARPU_ATOMIC_SOMETHING(add, old + value)
471 #define STARPU_ATOMIC_ADD(ptr, value) starpu_atomic_add(ptr, value) 473 #if defined(STARPU_HAVE_CMPXCHGL) 474 STARPU_ATOMIC_SOMETHINGL(add, old + value)
475 #define STARPU_ATOMIC_ADDL(ptr, value) starpu_atomic_addl(ptr, value) 477 #if defined(STARPU_HAVE_CMPXCHG64) 478 STARPU_ATOMIC_SOMETHING64(add, old + value)
479 #define STARPU_ATOMIC_ADD64(ptr, value) starpu_atomic_add64(ptr, value) 483 #ifdef STARPU_HAVE_SYNC_FETCH_AND_OR 484 #define STARPU_ATOMIC_OR(ptr, value) (__sync_fetch_and_or ((ptr), (value))) 485 #define STARPU_ATOMIC_ORL(ptr, value) (__sync_fetch_and_or ((ptr), (value))) 486 #define STARPU_ATOMIC_OR64(ptr, value) (__sync_fetch_and_or ((ptr), (value))) 488 #if defined(STARPU_HAVE_CMPXCHG) 489 STARPU_ATOMIC_SOMETHING(or, old | value)
490 #define STARPU_ATOMIC_OR(ptr, value) starpu_atomic_or(ptr, value) 492 #if defined(STARPU_HAVE_CMPXCHGL) 493 STARPU_ATOMIC_SOMETHINGL(or, old | value)
494 #define STARPU_ATOMIC_ORL(ptr, value) starpu_atomic_orl(ptr, value) 496 #if defined(STARPU_HAVE_CMPXCHG64) 497 STARPU_ATOMIC_SOMETHING64(or, old | value)
498 #define STARPU_ATOMIC_OR64(ptr, value) starpu_atomic_or64(ptr, value) 503 #ifdef STARPU_HAVE_SYNC_BOOL_COMPARE_AND_SWAP 504 #define STARPU_BOOL_COMPARE_AND_SWAP(ptr, old, value) (__sync_bool_compare_and_swap ((ptr), (old), (value))) 505 #define STARPU_BOOL_COMPARE_AND_SWAP32(ptr, old, value) STARPU_BOOL_COMPARE_AND_SWAP(ptr, old, value) 506 #define STARPU_BOOL_COMPARE_AND_SWAP64(ptr, old, value) STARPU_BOOL_COMPARE_AND_SWAP(ptr, old, value) 508 #ifdef STARPU_HAVE_CMPXCHG 509 #define STARPU_BOOL_COMPARE_AND_SWAP(ptr, old, value) (_starpu_cmpxchg((ptr), (old), (value)) == (old)) 511 #ifdef STARPU_HAVE_CMPXCHG32 512 #define STARPU_BOOL_COMPARE_AND_SWAP32(ptr, old, value) (_starpu_cmpxchg32((ptr), (old), (value)) == (old)) 514 #ifdef STARPU_HAVE_CMPXCHG64 515 #define STARPU_BOOL_COMPARE_AND_SWAP64(ptr, old, value) (_starpu_cmpxchg64((ptr), (old), (value)) == (old)) 519 #if UINTPTR_MAX == UINT64_MAX 520 #define STARPU_BOOL_COMPARE_AND_SWAP_PTR(ptr, old, value) STARPU_BOOL_COMPARE_AND_SWAP64(ptr, old, value) 522 #define STARPU_BOOL_COMPARE_AND_SWAP_PTR(ptr, old, value) STARPU_BOOL_COMPARE_AND_SWAP32(ptr, old, value) 526 #ifdef STARPU_HAVE_SYNC_VAL_COMPARE_AND_SWAP 527 #define STARPU_VAL_COMPARE_AND_SWAP(ptr, old, value) (__sync_val_compare_and_swap ((ptr), (old), (value))) 528 #define STARPU_VAL_COMPARE_AND_SWAP32(ptr, old, value) STARPU_VAL_COMPARE_AND_SWAP(ptr, old, value) 529 #define STARPU_VAL_COMPARE_AND_SWAP64(ptr, old, value) STARPU_VAL_COMPARE_AND_SWAP(ptr, old, value) 531 #ifdef STARPU_HAVE_CMPXCHG 532 #define STARPU_VAL_COMPARE_AND_SWAP(ptr, old, value) (_starpu_cmpxchg((ptr), (old), (value))) 534 #ifdef STARPU_HAVE_CMPXCHG32 535 #define STARPU_VAL_COMPARE_AND_SWAP32(ptr, old, value) (_starpu_cmpxchg32((ptr), (old), (value))) 537 #ifdef STARPU_HAVE_CMPXCHG64 538 #define STARPU_VAL_COMPARE_AND_SWAP64(ptr, old, value) (_starpu_cmpxchg64((ptr), (old), (value))) 542 #if UINTPTR_MAX == UINT64_MAX 543 #define STARPU_VAL_COMPARE_AND_SWAP_PTR(ptr, old, value) STARPU_VAL_COMPARE_AND_SWAP64(ptr, old, value) 545 #define STARPU_VAL_COMPARE_AND_SWAP_PTR(ptr, old, value) STARPU_VAL_COMPARE_AND_SWAP32(ptr, old, value) 548 #ifdef STARPU_HAVE_ATOMIC_EXCHANGE_N 549 #define STARPU_VAL_EXCHANGE(ptr, value) (__atomic_exchange_n((ptr), (value), __ATOMIC_SEQ_CST)) 550 #define STARPU_VAL_EXCHANGEL(ptr, value) STARPU_VAL_EXCHANGE((ptr) (value)) 551 #define STARPU_VAL_EXCHANGE32(ptr, value) STARPU_VAL_EXCHANGE((ptr) (value)) 552 #define STARPU_VAL_EXCHANGE64(ptr, value) STARPU_VAL_EXCHANGE((ptr) (value)) 554 #ifdef STARPU_HAVE_XCHG 555 #define STARPU_VAL_EXCHANGE(ptr, value) (_starpu_xchg((ptr), (value))) 557 #ifdef STARPU_HAVE_XCHGL 558 #define STARPU_VAL_EXCHANGEL(ptr, value) (_starpu_xchgl((ptr), (value))) 560 #ifdef STARPU_HAVE_XCHG32 561 #define STARPU_VAL_EXCHANGE32(ptr, value) (_starpu_xchg32((ptr), (value))) 563 #ifdef STARPU_HAVE_XCHG64 564 #define STARPU_VAL_EXCHANGE64(ptr, value) (_starpu_xchg64((ptr), (value))) 569 #ifdef STARPU_HAVE_SYNC_LOCK_TEST_AND_SET 570 #define STARPU_TEST_AND_SET(ptr, value) (__sync_lock_test_and_set ((ptr), (value))) 571 #define STARPU_RELEASE(ptr) (__sync_lock_release ((ptr))) 572 #elif defined(STARPU_HAVE_XCHG) 573 #define STARPU_TEST_AND_SET(ptr, value) (_starpu_xchg((ptr), (value))) 574 #define STARPU_RELEASE(ptr) (_starpu_xchg((ptr), 0)) 577 #ifdef STARPU_HAVE_SYNC_SYNCHRONIZE 578 #define STARPU_SYNCHRONIZE() __sync_synchronize() 579 #elif defined(__i386__) 580 #define STARPU_SYNCHRONIZE() __asm__ __volatile__("lock; addl $0,0(%%esp)" ::: "memory") 581 #elif defined(__KNC__) || defined(__KNF__) 582 #define STARPU_SYNCHRONIZE() __asm__ __volatile__("lock; addl $0,0(%%rsp)" ::: "memory") 583 #elif defined(__x86_64__) 584 #define STARPU_SYNCHRONIZE() __asm__ __volatile__("mfence" ::: "memory") 585 #elif defined(__ppc__) || defined(__ppc64__) 586 #define STARPU_SYNCHRONIZE() __asm__ __volatile__("sync" ::: "memory") 592 #if defined(__x86_64__) 593 #define STARPU_RMB() __asm__ __volatile__("lfence" ::: "memory") 594 #elif defined(__aarch64__) 595 #define STARPU_RMB() __asm__ __volatile__("dsb ld" ::: "memory") 597 #define STARPU_RMB() STARPU_SYNCHRONIZE() 603 #if defined(__x86_64__) 604 #define STARPU_WMB() __asm__ __volatile__("sfence" ::: "memory") 605 #elif defined(__aarch64__) 606 #define STARPU_WMB() __asm__ __volatile__("dsb st" ::: "memory") 608 #define STARPU_WMB() STARPU_SYNCHRONIZE() 611 #if defined(__i386__) || defined(__x86_64__) 612 #define STARPU_CACHELINE_SIZE 64 613 #elif defined(__ppc__) || defined(__ppc64__) || defined(__ia64__) 614 #define STARPU_CACHELINE_SIZE 128 615 #elif defined(__s390__) || defined(__s390x__) 616 #define STARPU_CACHELINE_SIZE 256 619 #define STARPU_CACHELINE_SIZE 1024 624 #include <sys/types.h> 625 #include <sys/stat.h> 630 #if !defined(_MSC_VER) || defined(BUILDING_STARPU) 633 #if !defined(STARPU_HAVE_STRUCT_TIMESPEC) || (defined(_MSC_VER) && _MSC_VER < 1900) 635 #ifndef STARPU_TIMESPEC_DEFINED 636 #define STARPU_TIMESPEC_DEFINED 1 645 #if defined(__MINGW32__) || defined(__CYGWIN__) 646 #include <sys/time.h> 649 #include <sys/time.h>