From 807d63a841e31ce7fa9c110aa78dc3094d9417b7 Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Mon, 13 Nov 2023 21:15:44 -0800 Subject: [PATCH 01/10] Update compiler-rt to LLVM 17.0.4 --- .../include/sanitizer/allocator_interface.h | 8 + .../include/sanitizer/common_interface_defs.h | 21 +- .../include/sanitizer/hwasan_interface.h | 2 +- .../include/sanitizer/tsan_interface.h | 123 +++++ .../compiler-rt/lib/asan/asan_allocator.cpp | 171 +++++-- .../lib/compiler-rt/lib/asan/asan_allocator.h | 9 +- .../lib/compiler-rt/lib/asan/asan_globals.cpp | 9 + .../lib/asan/asan_interceptors.cpp | 219 +++++--- .../compiler-rt/lib/asan/asan_interceptors.h | 20 +- .../asan/asan_interceptors_memintrinsics.cpp | 63 +++ .../asan/asan_interceptors_memintrinsics.h | 37 -- .../lib/asan/asan_interface_internal.h | 7 +- system/lib/compiler-rt/lib/asan/asan_mac.cpp | 50 +- .../compiler-rt/lib/asan/asan_malloc_win.cpp | 8 +- .../lib/asan/asan_memory_profile.cpp | 19 +- .../compiler-rt/lib/asan/asan_poisoning.cpp | 6 +- system/lib/compiler-rt/lib/asan/asan_stack.h | 32 +- .../lib/compiler-rt/lib/asan/asan_thread.cpp | 153 +++--- system/lib/compiler-rt/lib/asan/asan_thread.h | 6 +- system/lib/compiler-rt/lib/asan/asan_win.cpp | 5 + .../lib/asan/asan_win_dll_thunk.cpp | 2 + system/lib/compiler-rt/lib/builtins/absvdi2.c | 2 +- system/lib/compiler-rt/lib/builtins/absvsi2.c | 2 +- system/lib/compiler-rt/lib/builtins/addtf3.c | 2 +- system/lib/compiler-rt/lib/builtins/ashlti3.c | 2 +- system/lib/compiler-rt/lib/builtins/ashrti3.c | 2 +- .../lib/compiler-rt/lib/builtins/assembly.h | 2 +- .../compiler-rt/lib/builtins/clear_cache.c | 8 +- .../lib/compiler-rt/lib/builtins/comparetf2.c | 2 +- .../lib/compiler-rt/lib/builtins/cpu_model.c | 83 +++- .../lib/compiler-rt/lib/builtins/crtbegin.c | 135 +++++ system/lib/compiler-rt/lib/builtins/crtend.c | 22 + system/lib/compiler-rt/lib/builtins/divtf3.c | 2 +- .../compiler-rt/lib/builtins/extenddftf2.c | 2 +- .../compiler-rt/lib/builtins/extendhftf2.c | 3 +- .../compiler-rt/lib/builtins/extendsftf2.c | 2 +- system/lib/compiler-rt/lib/builtins/fixtfdi.c | 2 +- system/lib/compiler-rt/lib/builtins/fixtfsi.c | 2 +- system/lib/compiler-rt/lib/builtins/fixtfti.c | 2 +- .../lib/compiler-rt/lib/builtins/fixunstfdi.c | 2 +- .../lib/compiler-rt/lib/builtins/fixunstfsi.c | 2 +- .../lib/compiler-rt/lib/builtins/fixunstfti.c | 2 +- .../lib/compiler-rt/lib/builtins/floatdidf.c | 2 +- .../lib/compiler-rt/lib/builtins/floatdisf.c | 2 +- .../lib/compiler-rt/lib/builtins/floatditf.c | 2 +- .../lib/compiler-rt/lib/builtins/floatsitf.c | 2 +- .../lib/compiler-rt/lib/builtins/floattidf.c | 2 +- .../lib/compiler-rt/lib/builtins/floattisf.c | 2 +- .../lib/compiler-rt/lib/builtins/floattitf.c | 16 +- .../compiler-rt/lib/builtins/floatunditf.c | 2 +- .../compiler-rt/lib/builtins/floatunsitf.c | 2 +- .../compiler-rt/lib/builtins/floatuntidf.c | 2 +- .../compiler-rt/lib/builtins/floatuntisf.c | 2 +- .../compiler-rt/lib/builtins/floatuntitf.c | 26 +- system/lib/compiler-rt/lib/builtins/fp_lib.h | 7 + .../lib/builtins/gcc_personality_v0.c | 2 +- .../lib/builtins/int_mulo_impl.inc | 4 +- .../lib/builtins/int_mulv_impl.inc | 2 +- system/lib/compiler-rt/lib/builtins/lshrti3.c | 2 +- system/lib/compiler-rt/lib/builtins/mulodi4.c | 1 + system/lib/compiler-rt/lib/builtins/mulosi4.c | 1 + system/lib/compiler-rt/lib/builtins/muloti4.c | 1 + system/lib/compiler-rt/lib/builtins/multf3.c | 2 +- system/lib/compiler-rt/lib/builtins/mulvdi3.c | 1 + system/lib/compiler-rt/lib/builtins/mulvsi3.c | 1 + system/lib/compiler-rt/lib/builtins/mulvti3.c | 1 + system/lib/compiler-rt/lib/builtins/negvdi2.c | 3 +- system/lib/compiler-rt/lib/builtins/negvsi2.c | 3 +- .../lib/builtins/os_version_check.c | 8 +- system/lib/compiler-rt/lib/builtins/powitf2.c | 2 +- system/lib/compiler-rt/lib/builtins/subtf3.c | 2 +- .../lib/compiler-rt/lib/builtins/trunctfdf2.c | 2 +- .../lib/compiler-rt/lib/builtins/trunctfhf2.c | 3 +- .../lib/compiler-rt/lib/builtins/trunctfsf2.c | 2 +- .../lib/interception/interception.h | 201 +++++--- .../lib/interception/interception_linux.cpp | 16 +- .../lib/interception/interception_linux.h | 18 +- .../lib/interception/interception_win.cpp | 63 ++- .../lib/interception/interception_win.h | 5 + system/lib/compiler-rt/lib/lsan/lsan.cpp | 4 +- .../compiler-rt/lib/lsan/lsan_allocator.cpp | 64 ++- .../lib/compiler-rt/lib/lsan/lsan_allocator.h | 11 +- .../lib/compiler-rt/lib/lsan/lsan_common.cpp | 154 +++--- system/lib/compiler-rt/lib/lsan/lsan_common.h | 32 +- .../compiler-rt/lib/lsan/lsan_common_mac.cpp | 35 +- .../lib/compiler-rt/lib/lsan/lsan_fuchsia.cpp | 3 +- .../lib/lsan/lsan_interceptors.cpp | 142 +++--- .../lib/compiler-rt/lib/lsan/lsan_linux.cpp | 9 +- system/lib/compiler-rt/lib/lsan/lsan_mac.cpp | 21 +- .../lib/compiler-rt/lib/lsan/lsan_posix.cpp | 3 +- .../lib/compiler-rt/lib/lsan/lsan_thread.cpp | 58 ++- system/lib/compiler-rt/lib/lsan/lsan_thread.h | 14 +- .../sanitizer_common/sanitizer_allocator.cpp | 6 +- .../sanitizer_common/sanitizer_allocator.h | 7 + .../sanitizer_allocator_combined.h | 6 +- .../sanitizer_allocator_interface.h | 4 + .../sanitizer_allocator_internal.h | 1 - .../sanitizer_allocator_primary32.h | 2 +- .../sanitizer_allocator_primary64.h | 4 +- .../sanitizer_allocator_secondary.h | 4 +- .../sanitizer_allocator_stats.h | 27 +- .../sanitizer_common/sanitizer_array_ref.h | 123 +++++ .../lib/sanitizer_common/sanitizer_asm.h | 46 +- .../lib/sanitizer_common/sanitizer_common.cpp | 20 + .../lib/sanitizer_common/sanitizer_common.h | 28 +- .../sanitizer_common_interceptors.inc | 468 ++++++------------ .../sanitizer_common_interceptors_format.inc | 16 +- ...izer_common_interceptors_memintrinsics.inc | 244 +++++++++ .../sanitizer_common_interface.inc | 6 + .../sanitizer_common_libcdep.cpp | 2 + .../sanitizer_common_syscalls.inc | 11 +- .../sanitizer_coverage_libcdep_new.cpp | 9 +- .../lib/sanitizer_common/sanitizer_file.h | 1 + .../sanitizer_flag_parser.cpp | 4 +- .../sanitizer_common/sanitizer_flag_parser.h | 2 +- .../lib/sanitizer_common/sanitizer_flat_map.h | 17 +- .../sanitizer_common/sanitizer_fuchsia.cpp | 6 + .../sanitizer_interceptors_ioctl_netbsd.inc | 2 - .../sanitizer_internal_defs.h | 18 +- .../lib/sanitizer_common/sanitizer_libc.cpp | 15 +- .../lib/sanitizer_common/sanitizer_libc.h | 24 +- .../lib/sanitizer_common/sanitizer_linux.cpp | 12 +- .../lib/sanitizer_common/sanitizer_linux.h | 4 + .../sanitizer_linux_libcdep.cpp | 2 +- .../lib/sanitizer_common/sanitizer_mac.cpp | 132 ++--- .../lib/sanitizer_common/sanitizer_mallinfo.h | 38 ++ .../sanitizer_platform_interceptors.h | 13 +- .../sanitizer_platform_limits_freebsd.cpp | 9 + .../sanitizer_platform_limits_freebsd.h | 19 +- .../sanitizer_platform_limits_netbsd.cpp | 2 - .../sanitizer_platform_limits_netbsd.h | 2 - .../sanitizer_platform_limits_posix.cpp | 27 +- .../sanitizer_platform_limits_posix.h | 19 +- .../lib/sanitizer_common/sanitizer_posix.cpp | 12 +- .../lib/sanitizer_common/sanitizer_posix.h | 5 +- .../sanitizer_posix_libcdep.cpp | 4 +- .../lib/sanitizer_common/sanitizer_procmaps.h | 24 +- .../sanitizer_procmaps_mac.cpp | 16 +- .../sanitizer_common/sanitizer_quarantine.h | 25 +- .../lib/sanitizer_common/sanitizer_range.cpp | 62 +++ .../lib/sanitizer_common/sanitizer_range.h | 40 ++ .../sanitizer_redefine_builtins.h | 52 ++ .../sanitizer_signal_interceptors.inc | 3 + .../sanitizer_common/sanitizer_stacktrace.h | 4 +- .../sanitizer_stacktrace_printer.cpp | 56 ++- .../sanitizer_stacktrace_printer.h | 9 +- .../sanitizer_stacktrace_sparc.cpp | 6 - .../sanitizer_stoptheworld_mac.cpp | 6 +- .../sanitizer_suppressions.cpp | 1 + .../sanitizer_symbolizer_internal.h | 2 +- .../sanitizer_symbolizer_libbacktrace.cpp | 4 +- .../sanitizer_symbolizer_mac.cpp | 20 +- .../sanitizer_symbolizer_markup.cpp | 2 +- .../sanitizer_symbolizer_posix_libcdep.cpp | 38 +- .../sanitizer_symbolizer_report.cpp | 3 +- .../sanitizer_symbolizer_win.cpp | 14 +- .../sanitizer_thread_arg_retval.cpp | 94 ++++ .../sanitizer_thread_arg_retval.h | 116 +++++ .../sanitizer_thread_registry.cpp | 2 +- .../sanitizer_tls_get_addr.cpp | 29 +- .../sanitizer_common/sanitizer_tls_get_addr.h | 26 +- .../sanitizer_unwind_linux_libcdep.cpp | 6 - .../lib/sanitizer_common/sanitizer_win.cpp | 18 +- .../sanitizer_win_dll_thunk.h | 2 +- .../lib/compiler-rt/lib/ubsan/ubsan_diag.cpp | 7 +- .../compiler-rt/lib/ubsan/ubsan_handlers.cpp | 35 ++ .../compiler-rt/lib/ubsan/ubsan_handlers.h | 11 + .../lib/ubsan/ubsan_handlers_cxx.cpp | 44 -- .../lib/ubsan/ubsan_handlers_cxx.h | 16 - .../compiler-rt/lib/ubsan/ubsan_interface.inc | 4 +- .../lib/ubsan/ubsan_signals_standalone.cpp | 5 + 171 files changed, 3045 insertions(+), 1398 deletions(-) create mode 100644 system/lib/compiler-rt/lib/builtins/crtbegin.c create mode 100644 system/lib/compiler-rt/lib/builtins/crtend.c create mode 100644 system/lib/compiler-rt/lib/sanitizer_common/sanitizer_array_ref.h create mode 100644 system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc create mode 100644 system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mallinfo.h create mode 100644 system/lib/compiler-rt/lib/sanitizer_common/sanitizer_range.cpp create mode 100644 system/lib/compiler-rt/lib/sanitizer_common/sanitizer_range.h create mode 100644 system/lib/compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h create mode 100644 system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_arg_retval.cpp create mode 100644 system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_arg_retval.h diff --git a/system/lib/compiler-rt/include/sanitizer/allocator_interface.h b/system/lib/compiler-rt/include/sanitizer/allocator_interface.h index 6226135ef84b3..367e6409258f1 100644 --- a/system/lib/compiler-rt/include/sanitizer/allocator_interface.h +++ b/system/lib/compiler-rt/include/sanitizer/allocator_interface.h @@ -26,10 +26,18 @@ extern "C" { is not yet freed. */ int __sanitizer_get_ownership(const volatile void *p); + /* If a pointer lies within an allocation, it will return the start address + of the allocation. Otherwise, it returns nullptr. */ + const void *__sanitizer_get_allocated_begin(const void *p); + /* Returns the number of bytes reserved for the pointer p. Requires (get_ownership(p) == true) or (p == 0). */ size_t __sanitizer_get_allocated_size(const volatile void *p); + /* Returns the number of bytes reserved for the pointer p. + Requires __sanitizer_get_allocated_begin(p) == p. */ + size_t __sanitizer_get_allocated_size_fast(const volatile void *p); + /* Number of bytes, allocated and not yet freed by the application. */ size_t __sanitizer_get_current_allocated_bytes(void); diff --git a/system/lib/compiler-rt/include/sanitizer/common_interface_defs.h b/system/lib/compiler-rt/include/sanitizer/common_interface_defs.h index 2f415bd9e8540..983df7cea16ed 100644 --- a/system/lib/compiler-rt/include/sanitizer/common_interface_defs.h +++ b/system/lib/compiler-rt/include/sanitizer/common_interface_defs.h @@ -129,26 +129,23 @@ int __sanitizer_acquire_crash_state(); /// state mid == end, so that should be the final state when the /// container is destroyed or when the container reallocates the storage. /// -/// For ASan, beg should be 8-aligned and end -/// should be either 8-aligned or it should point to the end of a separate -/// heap-, stack-, or global-allocated buffer. So the following example will -/// not work: +/// For ASan, beg no longer needs to be 8-aligned, +/// first and last granule may be shared with other objects +/// and therefore the function can be used for any allocator. /// -/// \code -/// int64_t x[2]; // 16 bytes, 8-aligned -/// char *beg = (char *)&x[0]; -/// char *end = beg + 12; // Not 8-aligned, not the end of the buffer -/// \endcode +/// The following example shows how to use the function: /// -/// The following, however, will work: /// \code -/// int32_t x[3]; // 12 bytes, but 8-aligned under ASan. +/// int32_t x[3]; // 12 bytes /// char *beg = (char*)&x[0]; -/// char *end = beg + 12; // Not 8-aligned, but is the end of the buffer +/// char *end = beg + 12; +/// __sanitizer_annotate_contiguous_container(beg, end, beg, end); /// \endcode /// /// \note Use this function with caution and do not use for anything other /// than vector-like classes. +/// \note Unaligned beg or end may miss bugs in +/// these granules. /// /// \param beg Beginning of memory region. /// \param end End of memory region. diff --git a/system/lib/compiler-rt/include/sanitizer/hwasan_interface.h b/system/lib/compiler-rt/include/sanitizer/hwasan_interface.h index 14035c05c6353..ee742c7f30317 100644 --- a/system/lib/compiler-rt/include/sanitizer/hwasan_interface.h +++ b/system/lib/compiler-rt/include/sanitizer/hwasan_interface.h @@ -1,4 +1,4 @@ -//===-- sanitizer/asan_interface.h ------------------------------*- C++ -*-===// +//===-- sanitizer/hwasan_interface.h ----------------------------*- C++ -*-===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/system/lib/compiler-rt/include/sanitizer/tsan_interface.h b/system/lib/compiler-rt/include/sanitizer/tsan_interface.h index 2782e61fb8c79..f19c79d79ba62 100644 --- a/system/lib/compiler-rt/include/sanitizer/tsan_interface.h +++ b/system/lib/compiler-rt/include/sanitizer/tsan_interface.h @@ -172,6 +172,129 @@ int __tsan_on_finalize(int failed); // Release TSan internal memory in a best-effort manner. void __tsan_flush_memory(); +// User-provided default TSAN options. +const char* __tsan_default_options(void); + +// User-provided default TSAN suppressions. +const char* __tsan_default_suppressions(void); + +/// Returns a report's description. +/// +/// Returns a report's description (issue type), number of duplicate issues +/// found, counts of array data (stack traces, memory operations, locations, +/// mutexes, threads, unique thread IDs) and a stack trace of a sleep() +/// call (if one was involved in the issue). +/// +/// \param report Opaque pointer to the current report. +/// \param[out] description Report type description. +/// \param[out] count Count of duplicate issues. +/// \param[out] stack_count Count of stack traces. +/// \param[out] mop_count Count of memory operations. +/// \param[out] loc_count Count of locations. +/// \param[out] mutex_count Count of mutexes. +/// \param[out] thread_count Count of threads. +/// \param[out] unique_tid_count Count of unique thread IDs. +/// \param sleep_trace A buffer to store the stack trace of a sleep() +/// call. +/// \param trace_size Size in bytes of the trace buffer. +/// \returns Returns 1 if successful, 0 if not. +int __tsan_get_report_data(void *report, const char **description, int *count, + int *stack_count, int *mop_count, int *loc_count, + int *mutex_count, int *thread_count, + int *unique_tid_count, void **sleep_trace, + unsigned long trace_size); + +/// Returns information about stack traces included in the report. +/// +/// \param report Opaque pointer to the current report. +/// \param idx Index to the report's stacks. +/// \param trace A buffer to store the stack trace. +/// \param trace_size Size in bytes of the trace buffer. +/// \returns Returns 1 if successful, 0 if not. +int __tsan_get_report_stack(void *report, unsigned long idx, void **trace, + unsigned long trace_size); + +/// Returns information about memory operations included in the report. +/// +/// \param report Opaque pointer to the current report. +/// \param idx Index to the report's memory operations. +/// \param[out] tid Thread ID of the memory operation. +/// \param[out] addr Address of the memory operation. +/// \param[out] size Size of the memory operation. +/// \param[out] write Write flag of the memory operation. +/// \param[out] atomic Atomicity flag of the memory operation. +/// \param trace A buffer to store the stack trace. +/// \param trace_size Size in bytes of the trace buffer. +/// \returns Returns 1 if successful, 0 if not. +int __tsan_get_report_mop(void *report, unsigned long idx, int *tid, + void **addr, int *size, int *write, int *atomic, + void **trace, unsigned long trace_size); + +/// Returns information about locations included in the report. +/// +/// \param report Opaque pointer to the current report. +/// \param idx Index to the report's locations. +/// \param[out] type Type of the location. +/// \param[out] addr Address of the location. +/// \param[out] start Start of the location. +/// \param[out] size Size of the location. +/// \param[out] tid Thread ID of the location. +/// \param[out] fd File descriptor of the location. +/// \param[out] suppressable Suppressable flag. +/// \param trace A buffer to store the stack trace. +/// \param trace_size Size in bytes of the trace buffer. +/// \returns Returns 1 if successful, 0 if not. +int __tsan_get_report_loc(void *report, unsigned long idx, const char **type, + void **addr, void **start, unsigned long *size, + int *tid, int *fd, int *suppressable, void **trace, + unsigned long trace_size); + +/// Returns information about mutexes included in the report. +/// +/// \param report Opaque pointer to the current report. +/// \param idx Index to the report's mutexes. +/// \param[out] mutex_id Id of the mutex. +/// \param[out] addr Address of the mutex. +/// \param[out] destroyed Destroyed mutex flag. +/// \param trace A buffer to store the stack trace. +/// \param trace_size Size in bytes of the trace buffer. +/// \returns Returns 1 if successful, 0 if not. +int __tsan_get_report_mutex(void *report, unsigned long idx, uint64_t *mutex_id, + void **addr, int *destroyed, void **trace, + unsigned long trace_size); + +/// Returns information about threads included in the report. +/// +/// \param report Opaque pointer to the current report. +/// \param idx Index to the report's threads. +/// \param[out] tid Thread ID of the thread. +/// \param[out] os_id Operating system's ID of the thread. +/// \param[out] running Running flag of the thread. +/// \param[out] name Name of the thread. +/// \param[out] parent_tid ID of the parent thread. +/// \param trace A buffer to store the stack trace. +/// \param trace_size Size in bytes of the trace buffer. +/// \returns Returns 1 if successful, 0 if not. +int __tsan_get_report_thread(void *report, unsigned long idx, int *tid, + uint64_t *os_id, int *running, const char **name, + int *parent_tid, void **trace, + unsigned long trace_size); + +/// Returns information about unique thread IDs included in the report. +/// +/// \param report Opaque pointer to the current report. +/// \param idx Index to the report's unique thread IDs. +/// \param[out] tid Unique thread ID of the report. +/// \returns Returns 1 if successful, 0 if not. +int __tsan_get_report_unique_tid(void *report, unsigned long idx, int *tid); + +/// Returns the current report. +/// +/// If TSan is currently reporting a detected issue on the current thread, +/// returns an opaque pointer to the current report. Otherwise returns NULL. +/// \returns An opaque pointer to the current report. Otherwise returns NULL. +void *__tsan_get_current_report(); + #ifdef __cplusplus } // extern "C" #endif diff --git a/system/lib/compiler-rt/lib/asan/asan_allocator.cpp b/system/lib/compiler-rt/lib/asan/asan_allocator.cpp index 74183fcd24273..15eefcb96063a 100644 --- a/system/lib/compiler-rt/lib/asan/asan_allocator.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_allocator.cpp @@ -16,6 +16,7 @@ #include "asan_allocator.h" +#include "asan_internal.h" #include "asan_mapping.h" #include "asan_poisoning.h" #include "asan_report.h" @@ -24,6 +25,7 @@ #include "lsan/lsan_common.h" #include "sanitizer_common/sanitizer_allocator_checks.h" #include "sanitizer_common/sanitizer_allocator_interface.h" +#include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_errno.h" #include "sanitizer_common/sanitizer_flags.h" #include "sanitizer_common/sanitizer_internal_defs.h" @@ -190,28 +192,56 @@ class LargeChunkHeader { } }; +static void FillChunk(AsanChunk *m) { + // FIXME: Use ReleaseMemoryPagesToOS. + Flags &fl = *flags(); + + if (fl.max_free_fill_size > 0) { + // We have to skip the chunk header, it contains free_context_id. + uptr scribble_start = (uptr)m + kChunkHeaderSize + kChunkHeader2Size; + if (m->UsedSize() >= kChunkHeader2Size) { // Skip Header2 in user area. + uptr size_to_fill = m->UsedSize() - kChunkHeader2Size; + size_to_fill = Min(size_to_fill, (uptr)fl.max_free_fill_size); + REAL(memset)((void *)scribble_start, fl.free_fill_byte, size_to_fill); + } + } +} + struct QuarantineCallback { QuarantineCallback(AllocatorCache *cache, BufferedStackTrace *stack) : cache_(cache), stack_(stack) { } - void Recycle(AsanChunk *m) { + void PreQuarantine(AsanChunk *m) const { + FillChunk(m); + // Poison the region. + PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), ASAN_SHADOW_GRANULARITY), + kAsanHeapFreeMagic); + } + + void Recycle(AsanChunk *m) const { void *p = get_allocator().GetBlockBegin(m); - if (p != m) { - // Clear the magic value, as allocator internals may overwrite the - // contents of deallocated chunk, confusing GetAsanChunk lookup. - reinterpret_cast(p)->Set(nullptr); - } - u8 old_chunk_state = CHUNK_QUARANTINE; - if (!atomic_compare_exchange_strong(&m->chunk_state, &old_chunk_state, - CHUNK_INVALID, memory_order_acquire)) { - CHECK_EQ(old_chunk_state, CHUNK_QUARANTINE); - } + // The secondary will immediately unpoison and unmap the memory, so this + // branch is unnecessary. + if (get_allocator().FromPrimary(p)) { + if (p != m) { + // Clear the magic value, as allocator internals may overwrite the + // contents of deallocated chunk, confusing GetAsanChunk lookup. + reinterpret_cast(p)->Set(nullptr); + } - PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), ASAN_SHADOW_GRANULARITY), - kAsanHeapLeftRedzoneMagic); + u8 old_chunk_state = CHUNK_QUARANTINE; + if (!atomic_compare_exchange_strong(&m->chunk_state, &old_chunk_state, + CHUNK_INVALID, + memory_order_acquire)) { + CHECK_EQ(old_chunk_state, CHUNK_QUARANTINE); + } + + PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), ASAN_SHADOW_GRANULARITY), + kAsanHeapLeftRedzoneMagic); + } // Statistics. AsanStats &thread_stats = GetCurrentThreadStats(); @@ -221,7 +251,17 @@ struct QuarantineCallback { get_allocator().Deallocate(cache_, p); } - void *Allocate(uptr size) { + void RecyclePassThrough(AsanChunk *m) const { + // Recycle for the secondary will immediately unpoison and unmap the + // memory, so quarantine preparation is unnecessary. + if (get_allocator().FromPrimary(m)) { + // The primary allocation may need pattern fill if enabled. + FillChunk(m); + } + Recycle(m); + } + + void *Allocate(uptr size) const { void *res = get_allocator().Allocate(cache_, size, 1); // TODO(alekseys): Consider making quarantine OOM-friendly. if (UNLIKELY(!res)) @@ -229,9 +269,7 @@ struct QuarantineCallback { return res; } - void Deallocate(void *p) { - get_allocator().Deallocate(cache_, p); - } + void Deallocate(void *p) const { get_allocator().Deallocate(cache_, p); } private: AllocatorCache* const cache_; @@ -248,6 +286,22 @@ void AsanMapUnmapCallback::OnMap(uptr p, uptr size) const { thread_stats.mmaps++; thread_stats.mmaped += size; } + +void AsanMapUnmapCallback::OnMapSecondary(uptr p, uptr size, uptr user_begin, + uptr user_size) const { + uptr user_end = RoundDownTo(user_begin + user_size, ASAN_SHADOW_GRANULARITY); + user_begin = RoundUpTo(user_begin, ASAN_SHADOW_GRANULARITY); + // The secondary mapping will be immediately returned to user, no value + // poisoning that with non-zero just before unpoisoning by Allocate(). So just + // poison head/tail invisible to Allocate(). + PoisonShadow(p, user_begin - p, kAsanHeapLeftRedzoneMagic); + PoisonShadow(user_end, size - (user_end - p), kAsanHeapLeftRedzoneMagic); + // Statistics. + AsanStats &thread_stats = GetCurrentThreadStats(); + thread_stats.mmaps++; + thread_stats.mmaped += size; +} + void AsanMapUnmapCallback::OnUnmap(uptr p, uptr size) const { PoisonShadow(p, size, 0); // We are about to unmap a chunk of user memory. @@ -387,8 +441,9 @@ struct Allocator { } void GetOptions(AllocatorOptions *options) const { - options->quarantine_size_mb = quarantine.GetSize() >> 20; - options->thread_local_quarantine_size_kb = quarantine.GetCacheSize() >> 10; + options->quarantine_size_mb = quarantine.GetMaxSize() >> 20; + options->thread_local_quarantine_size_kb = + quarantine.GetMaxCacheSize() >> 10; options->min_redzone = atomic_load(&min_redzone, memory_order_acquire); options->max_redzone = atomic_load(&max_redzone, memory_order_acquire); options->may_return_null = AllocatorMayReturnNull(); @@ -502,9 +557,10 @@ struct Allocator { uptr needed_size = rounded_size + rz_size; if (alignment > min_alignment) needed_size += alignment; + bool from_primary = PrimaryAllocator::CanAllocate(needed_size, alignment); // If we are allocating from the secondary allocator, there will be no // automatic right redzone, so add the right redzone manually. - if (!PrimaryAllocator::CanAllocate(needed_size, alignment)) + if (!from_primary) needed_size += rz_size; CHECK(IsAligned(needed_size, min_alignment)); if (size > kMaxAllowedMallocSize || needed_size > kMaxAllowedMallocSize || @@ -536,15 +592,6 @@ struct Allocator { ReportOutOfMemory(size, stack); } - if (*(u8 *)MEM_TO_SHADOW((uptr)allocated) == 0 && CanPoisonMemory()) { - // Heap poisoning is enabled, but the allocator provides an unpoisoned - // chunk. This is possible if CanPoisonMemory() was false for some - // time, for example, due to flags()->start_disabled. - // Anyway, poison the block before using it for anything else. - uptr allocated_size = allocator.GetActuallyAllocatedSize(allocated); - PoisonShadow((uptr)allocated, allocated_size, kAsanHeapLeftRedzoneMagic); - } - uptr alloc_beg = reinterpret_cast(allocated); uptr alloc_end = alloc_beg + needed_size; uptr user_beg = alloc_beg + rz_size; @@ -561,6 +608,17 @@ struct Allocator { m->SetAllocContext(t ? t->tid() : kMainTid, StackDepotPut(*stack)); + if (!from_primary || *(u8 *)MEM_TO_SHADOW((uptr)allocated) == 0) { + // The allocator provides an unpoisoned chunk. This is possible for the + // secondary allocator, or if CanPoisonMemory() was false for some time, + // for example, due to flags()->start_disabled. Anyway, poison left and + // right of the block before using it for anything else. + uptr tail_beg = RoundUpTo(user_end, ASAN_SHADOW_GRANULARITY); + uptr tail_end = alloc_beg + allocator.GetActuallyAllocatedSize(allocated); + PoisonShadow(alloc_beg, user_beg - alloc_beg, kAsanHeapLeftRedzoneMagic); + PoisonShadow(tail_beg, tail_end - tail_beg, kAsanHeapLeftRedzoneMagic); + } + uptr size_rounded_down_to_granularity = RoundDownTo(size, ASAN_SHADOW_GRANULARITY); // Unpoison the bulk of the memory region. @@ -628,25 +686,6 @@ struct Allocator { AsanThread *t = GetCurrentThread(); m->SetFreeContext(t ? t->tid() : 0, StackDepotPut(*stack)); - Flags &fl = *flags(); - if (fl.max_free_fill_size > 0) { - // We have to skip the chunk header, it contains free_context_id. - uptr scribble_start = (uptr)m + kChunkHeaderSize + kChunkHeader2Size; - if (m->UsedSize() >= kChunkHeader2Size) { // Skip Header2 in user area. - uptr size_to_fill = m->UsedSize() - kChunkHeader2Size; - size_to_fill = Min(size_to_fill, (uptr)fl.max_free_fill_size); - REAL(memset)((void *)scribble_start, fl.free_fill_byte, size_to_fill); - } - } - - // Poison the region. - PoisonShadow(m->Beg(), RoundUpTo(m->UsedSize(), ASAN_SHADOW_GRANULARITY), - kAsanHeapFreeMagic); - - AsanStats &thread_stats = GetCurrentThreadStats(); - thread_stats.frees++; - thread_stats.freed += m->UsedSize(); - // Push into quarantine. if (t) { AsanThreadLocalMallocStorage *ms = &t->malloc_storage(); @@ -699,6 +738,10 @@ struct Allocator { } } + AsanStats &thread_stats = GetCurrentThreadStats(); + thread_stats.frees++; + thread_stats.freed += m->UsedSize(); + QuarantineChunk(m, ptr, stack); } @@ -798,6 +841,10 @@ struct Allocator { return m->UsedSize(); } + uptr AllocationSizeFast(uptr p) { + return reinterpret_cast(p - kChunkHeaderSize)->UsedSize(); + } + AsanChunkView FindHeapChunkByAddress(uptr addr) { AsanChunk *m1 = GetAsanChunkByAddr(addr); sptr offset = 0; @@ -1100,6 +1147,10 @@ uptr GetUserBegin(uptr chunk) { return m ? m->Beg() : 0; } +uptr GetUserAddr(uptr chunk) { + return chunk; +} + LsanMetadata::LsanMetadata(uptr chunk) { metadata_ = chunk ? reinterpret_cast(chunk - __asan::kChunkHeaderSize) : nullptr; @@ -1140,7 +1191,7 @@ void ForEachChunk(ForEachChunkCallback callback, void *arg) { __asan::get_allocator().ForEachChunk(callback, arg); } -IgnoreObjectResult IgnoreObjectLocked(const void *p) { +IgnoreObjectResult IgnoreObject(const void *p) { uptr addr = reinterpret_cast(p); __asan::AsanChunk *m = __asan::instance.GetAsanChunkByAddr(addr); if (!m || @@ -1160,6 +1211,17 @@ IgnoreObjectResult IgnoreObjectLocked(const void *p) { // ---------------------- Interface ---------------- {{{1 using namespace __asan; +static const void *AllocationBegin(const void *p) { + AsanChunk *m = __asan::instance.GetAsanChunkByAddr((uptr)p); + if (!m) + return nullptr; + if (atomic_load(&m->chunk_state, memory_order_acquire) != CHUNK_ALLOCATED) + return nullptr; + if (m->UsedSize() == 0) + return nullptr; + return (const void *)(m->Beg()); +} + // ASan allocator doesn't reserve extra bytes, so normally we would // just return "size". We don't want to expose our redzone sizes, etc here. uptr __sanitizer_get_estimated_allocated_size(uptr size) { @@ -1183,6 +1245,17 @@ uptr __sanitizer_get_allocated_size(const void *p) { return allocated_size; } +uptr __sanitizer_get_allocated_size_fast(const void *p) { + DCHECK_EQ(p, __sanitizer_get_allocated_begin(p)); + uptr ret = instance.AllocationSizeFast(reinterpret_cast(p)); + DCHECK_EQ(ret, __sanitizer_get_allocated_size(p)); + return ret; +} + +const void *__sanitizer_get_allocated_begin(const void *p) { + return AllocationBegin(p); +} + void __sanitizer_purge_allocator() { GET_STACK_TRACE_MALLOC; instance.Purge(&stack); diff --git a/system/lib/compiler-rt/lib/asan/asan_allocator.h b/system/lib/compiler-rt/lib/asan/asan_allocator.h index 0b4dbf03bb9d5..ffeedadf0772d 100644 --- a/system/lib/compiler-rt/lib/asan/asan_allocator.h +++ b/system/lib/compiler-rt/lib/asan/asan_allocator.h @@ -114,6 +114,7 @@ class AsanChunkFifoList: public IntrusiveList { struct AsanMapUnmapCallback { void OnMap(uptr p, uptr size) const; + void OnMapSecondary(uptr p, uptr size, uptr user_begin, uptr user_size) const; void OnUnmap(uptr p, uptr size) const; }; @@ -143,11 +144,15 @@ typedef DefaultSizeClassMap SizeClassMap; const uptr kAllocatorSpace = ~(uptr)0; const uptr kAllocatorSize = 0x8000000000ULL; // 500G typedef DefaultSizeClassMap SizeClassMap; -# else +# elif SANITIZER_APPLE const uptr kAllocatorSpace = 0x600000000000ULL; const uptr kAllocatorSize = 0x40000000000ULL; // 4T. typedef DefaultSizeClassMap SizeClassMap; -# endif +# else +const uptr kAllocatorSpace = 0x500000000000ULL; +const uptr kAllocatorSize = 0x40000000000ULL; // 4T. +typedef DefaultSizeClassMap SizeClassMap; +# endif template struct AP64 { // Allocator64 parameters. Deliberately using a short name. static const uptr kSpaceBeg = kAllocatorSpace; diff --git a/system/lib/compiler-rt/lib/asan/asan_globals.cpp b/system/lib/compiler-rt/lib/asan/asan_globals.cpp index 80de7403a4bc3..a358c9cadf985 100644 --- a/system/lib/compiler-rt/lib/asan/asan_globals.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_globals.cpp @@ -92,6 +92,10 @@ static void ReportGlobal(const Global &g, const char *prefix) { if (info.line != 0) { Report(" location: name=%s, %d\n", info.file, static_cast(info.line)); } + else if (g.gcc_location != 0) { + // Fallback to Global::gcc_location + Report(" location: name=%s, %d\n", g.gcc_location->filename, g.gcc_location->line_no); + } } static u32 FindRegistrationSite(const Global *g) { @@ -302,6 +306,11 @@ void PrintGlobalLocation(InternalScopedString *str, const __asan_global &g) { if (info.line != 0) { str->append("%s:%d", info.file, static_cast(info.line)); + } else if (g.gcc_location != 0) { + // Fallback to Global::gcc_location + str->append("%s", g.gcc_location->filename ? g.gcc_location->filename : g.module_name); + if (g.gcc_location->line_no) str->append(":%d", g.gcc_location->line_no); + if (g.gcc_location->column_no) str->append(":%d", g.gcc_location->column_no); } else { str->append("%s", g.module_name); } diff --git a/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp b/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp index 2f0ef224e3d7e..12f5fe918d95c 100644 --- a/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "asan_interceptors.h" + #include "asan_allocator.h" #include "asan_internal.h" #include "asan_mapping.h" @@ -20,7 +21,10 @@ #include "asan_stack.h" #include "asan_stats.h" #include "asan_suppressions.h" +#include "asan_thread.h" #include "lsan/lsan_common.h" +#include "sanitizer_common/sanitizer_errno.h" +#include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_libc.h" // There is no general interception at all on Fuchsia or Emscripten. @@ -84,12 +88,6 @@ using namespace __asan; DECLARE_REAL_AND_INTERCEPTOR(void *, malloc, uptr) DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) -#define ASAN_INTERCEPTOR_ENTER(ctx, func) \ - AsanInterceptorContext _ctx = {#func}; \ - ctx = (void *)&_ctx; \ - (void) ctx; \ - -#define COMMON_INTERCEPT_FUNCTION(name) ASAN_INTERCEPT_FUNC(name) #define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \ ASAN_INTERCEPT_FUNC_VER(name, ver) #define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \ @@ -149,22 +147,46 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) *begin = *end = 0; \ } -#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size) \ - do { \ - ASAN_INTERCEPTOR_ENTER(ctx, memmove); \ - ASAN_MEMMOVE_IMPL(ctx, to, from, size); \ - } while (false) +template +static void* mmap_interceptor(Mmap real_mmap, void *addr, SIZE_T length, + int prot, int flags, int fd, OFF64_T offset) { + void *res = real_mmap(addr, length, prot, flags, fd, offset); + if (length && res != (void *)-1) { + const uptr beg = reinterpret_cast(res); + DCHECK(IsAligned(beg, GetPageSize())); + SIZE_T rounded_length = RoundUpTo(length, GetPageSize()); + // Only unpoison shadow if it's an ASAN managed address. + if (AddrIsInMem(beg) && AddrIsInMem(beg + rounded_length - 1)) + PoisonShadow(beg, RoundUpTo(length, GetPageSize()), 0); + } + return res; +} -#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size) \ - do { \ - ASAN_INTERCEPTOR_ENTER(ctx, memcpy); \ - ASAN_MEMCPY_IMPL(ctx, to, from, size); \ +template +static int munmap_interceptor(Munmap real_munmap, void *addr, SIZE_T length) { + // We should not tag if munmap fail, but it's to late to tag after + // real_munmap, as the pages could be mmaped by another thread. + const uptr beg = reinterpret_cast(addr); + if (length && IsAligned(beg, GetPageSize())) { + SIZE_T rounded_length = RoundUpTo(length, GetPageSize()); + // Protect from unmapping the shadow. + if (AddrIsInMem(beg) && AddrIsInMem(beg + rounded_length - 1)) + PoisonShadow(beg, rounded_length, 0); + } + return real_munmap(addr, length); +} + +# define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, length, prot, flags, \ + fd, offset) \ + do { \ + (void)(ctx); \ + return mmap_interceptor(REAL(mmap), addr, sz, prot, flags, fd, off); \ } while (false) -#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size) \ - do { \ - ASAN_INTERCEPTOR_ENTER(ctx, memset); \ - ASAN_MEMSET_IMPL(ctx, block, c, size); \ +# define COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, length) \ + do { \ + (void)(ctx); \ + return munmap_interceptor(REAL(munmap), addr, sz); \ } while (false) #if CAN_SANITIZE_LEAKS @@ -172,6 +194,8 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) __lsan::ScopedInterceptorDisabler disabler #endif +#define SIGNAL_INTERCEPTOR_ENTER() ENSURE_ASAN_INITED() + #include "sanitizer_common/sanitizer_common_interceptors.inc" #include "sanitizer_common/sanitizer_signal_interceptors.inc" @@ -196,19 +220,26 @@ DECLARE_REAL_AND_INTERCEPTOR(void, free, void *) static thread_return_t THREAD_CALLING_CONV asan_thread_start(void *arg) { AsanThread *t = (AsanThread *)arg; SetCurrentThread(t); - return t->ThreadStart(GetTid()); + auto self = GetThreadSelf(); + auto args = asanThreadArgRetval().GetArgs(self); + thread_return_t retval = t->ThreadStart(GetTid()); + asanThreadArgRetval().Finish(self, retval); + CHECK_EQ(args.arg_retval, t->get_arg()); + return retval; } -INTERCEPTOR(int, pthread_create, void *thread, - void *attr, void *(*start_routine)(void*), void *arg) { +INTERCEPTOR(int, pthread_create, void *thread, void *attr, + void *(*start_routine)(void *), void *arg) { EnsureMainThreadIDIsCorrect(); // Strict init-order checking is thread-hostile. if (flags()->strict_init_order) StopInitOrderChecking(); GET_STACK_TRACE_THREAD; - int detached = 0; - if (attr) - REAL(pthread_attr_getdetachstate)(attr, &detached); + bool detached = [attr]() { + int d = 0; + return attr && !REAL(pthread_attr_getdetachstate)(attr, &d) && + IsStateDetached(d); + }(); u32 current_tid = GetCurrentTidOrInvalid(); AsanThread *t = @@ -220,10 +251,13 @@ INTERCEPTOR(int, pthread_create, void *thread, // stored by pthread for future reuse even after thread destruction, and // the linked list it's stored in doesn't even hold valid pointers to the // objects, the latter are calculated by obscure pointer arithmetic. -#if CAN_SANITIZE_LEAKS +# if CAN_SANITIZE_LEAKS __lsan::ScopedInterceptorDisabler disabler; -#endif - result = REAL(pthread_create)(thread, attr, asan_thread_start, t); +# endif + asanThreadArgRetval().Create(detached, {start_routine, arg}, [&]() -> uptr { + result = REAL(pthread_create)(thread, attr, asan_thread_start, t); + return result ? 0 : *(uptr *)(thread); + }); } if (result != 0) { // If the thread didn't start delete the AsanThread to avoid leaking it. @@ -234,9 +268,51 @@ INTERCEPTOR(int, pthread_create, void *thread, return result; } -INTERCEPTOR(int, pthread_join, void *t, void **arg) { - return real_pthread_join(t, arg); +INTERCEPTOR(int, pthread_join, void *thread, void **retval) { + int result; + asanThreadArgRetval().Join((uptr)thread, [&]() { + result = REAL(pthread_join)(thread, retval); + return !result; + }); + return result; +} + +INTERCEPTOR(int, pthread_detach, void *thread) { + int result; + asanThreadArgRetval().Detach((uptr)thread, [&]() { + result = REAL(pthread_detach)(thread); + return !result; + }); + return result; +} + +INTERCEPTOR(int, pthread_exit, void *retval) { + asanThreadArgRetval().Finish(GetThreadSelf(), retval); + return REAL(pthread_exit)(retval); +} + +# if ASAN_INTERCEPT_TRYJOIN +INTERCEPTOR(int, pthread_tryjoin_np, void *thread, void **ret) { + int result; + asanThreadArgRetval().Join((uptr)thread, [&]() { + result = REAL(pthread_tryjoin_np)(thread, ret); + return !result; + }); + return result; +} +# endif + +# if ASAN_INTERCEPT_TIMEDJOIN +INTERCEPTOR(int, pthread_timedjoin_np, void *thread, void **ret, + const struct timespec *abstime) { + int result; + asanThreadArgRetval().Join((uptr)thread, [&]() { + result = REAL(pthread_timedjoin_np)(thread, ret, abstime); + return !result; + }); + return result; } +# endif DEFINE_REAL_PTHREAD_FUNCTIONS #endif // ASAN_INTERCEPT_PTHREAD_CREATE @@ -388,7 +464,7 @@ INTERCEPTOR(_Unwind_Reason_Code, _Unwind_SjLj_RaiseException, #if ASAN_INTERCEPT_INDEX # if ASAN_USE_ALIAS_ATTRIBUTE_FOR_INDEX INTERCEPTOR(char*, index, const char *string, int c) - ALIAS(WRAPPER_NAME(strchr)); + ALIAS(WRAP(strchr)); # else # if SANITIZER_APPLE DECLARE_REAL(char*, index, const char *string, int c) @@ -474,7 +550,9 @@ INTERCEPTOR(char*, strdup, const char *s) { } GET_STACK_TRACE_MALLOC; void *new_mem = asan_malloc(length + 1, &stack); - REAL(memcpy)(new_mem, s, length + 1); + if (new_mem) { + REAL(memcpy)(new_mem, s, length + 1); + } return reinterpret_cast(new_mem); } @@ -490,7 +568,9 @@ INTERCEPTOR(char*, __strdup, const char *s) { } GET_STACK_TRACE_MALLOC; void *new_mem = asan_malloc(length + 1, &stack); - REAL(memcpy)(new_mem, s, length + 1); + if (new_mem) { + REAL(memcpy)(new_mem, s, length + 1); + } return reinterpret_cast(new_mem); } #endif // ASAN_INTERCEPT___STRDUP @@ -508,19 +588,34 @@ INTERCEPTOR(char*, strncpy, char *to, const char *from, uptr size) { return REAL(strncpy)(to, from, size); } -INTERCEPTOR(long, strtol, const char *nptr, char **endptr, int base) { - void *ctx; - ASAN_INTERCEPTOR_ENTER(ctx, strtol); - ENSURE_ASAN_INITED(); - if (!flags()->replace_str) { - return REAL(strtol)(nptr, endptr, base); - } +template +static ALWAYS_INLINE auto StrtolImpl(void *ctx, Fn real, const char *nptr, + char **endptr, int base) + -> decltype(real(nullptr, nullptr, 0)) { + if (!flags()->replace_str) + return real(nptr, endptr, base); char *real_endptr; - long result = REAL(strtol)(nptr, &real_endptr, base); + auto res = real(nptr, &real_endptr, base); StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); - return result; + return res; } +# define INTERCEPTOR_STRTO_BASE(ret_type, func) \ + INTERCEPTOR(ret_type, func, const char *nptr, char **endptr, int base) { \ + void *ctx; \ + ASAN_INTERCEPTOR_ENTER(ctx, func); \ + ENSURE_ASAN_INITED(); \ + return StrtolImpl(ctx, REAL(func), nptr, endptr, base); \ + } + +INTERCEPTOR_STRTO_BASE(long, strtol) +INTERCEPTOR_STRTO_BASE(long long, strtoll) + +# if SANITIZER_GLIBC +INTERCEPTOR_STRTO_BASE(long, __isoc23_strtol) +INTERCEPTOR_STRTO_BASE(long long, __isoc23_strtoll) +# endif + INTERCEPTOR(int, atoi, const char *nptr) { void *ctx; ASAN_INTERCEPTOR_ENTER(ctx, atoi); @@ -559,20 +654,6 @@ INTERCEPTOR(long, atol, const char *nptr) { return result; } -#if ASAN_INTERCEPT_ATOLL_AND_STRTOLL -INTERCEPTOR(long long, strtoll, const char *nptr, char **endptr, int base) { - void *ctx; - ASAN_INTERCEPTOR_ENTER(ctx, strtoll); - ENSURE_ASAN_INITED(); - if (!flags()->replace_str) { - return REAL(strtoll)(nptr, endptr, base); - } - char *real_endptr; - long long result = REAL(strtoll)(nptr, &real_endptr, base); - StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); - return result; -} - INTERCEPTOR(long long, atoll, const char *nptr) { void *ctx; ASAN_INTERCEPTOR_ENTER(ctx, atoll); @@ -586,7 +667,6 @@ INTERCEPTOR(long long, atoll, const char *nptr) { ASAN_READ_STRING(ctx, nptr, (real_endptr - nptr) + 1); return result; } -#endif // ASAN_INTERCEPT_ATOLL_AND_STRTOLL #if ASAN_INTERCEPT___CXA_ATEXIT || ASAN_INTERCEPT_ATEXIT static void AtCxaAtexit(void *unused) { @@ -652,6 +732,7 @@ void InitializeAsanInterceptors() { static bool was_called_once; CHECK(!was_called_once); was_called_once = true; + InitializePlatformInterceptors(); InitializeCommonInterceptors(); InitializeSignalInterceptors(); @@ -670,11 +751,13 @@ void InitializeAsanInterceptors() { ASAN_INTERCEPT_FUNC(atoi); ASAN_INTERCEPT_FUNC(atol); - ASAN_INTERCEPT_FUNC(strtol); -#if ASAN_INTERCEPT_ATOLL_AND_STRTOLL ASAN_INTERCEPT_FUNC(atoll); + ASAN_INTERCEPT_FUNC(strtol); ASAN_INTERCEPT_FUNC(strtoll); -#endif +# if SANITIZER_GLIBC + ASAN_INTERCEPT_FUNC(__isoc23_strtol); + ASAN_INTERCEPT_FUNC(__isoc23_strtoll); +# endif // Intecept jump-related functions. ASAN_INTERCEPT_FUNC(longjmp); @@ -702,11 +785,11 @@ void InitializeAsanInterceptors() { #endif // Indirectly intercept std::rethrow_exception. #if ASAN_INTERCEPT__UNWIND_RAISEEXCEPTION - INTERCEPT_FUNCTION(_Unwind_RaiseException); + ASAN_INTERCEPT_FUNC(_Unwind_RaiseException); #endif // Indirectly intercept std::rethrow_exception. #if ASAN_INTERCEPT__UNWIND_SJLJ_RAISEEXCEPTION - INTERCEPT_FUNCTION(_Unwind_SjLj_RaiseException); + ASAN_INTERCEPT_FUNC(_Unwind_SjLj_RaiseException); #endif // Intercept threading-related functions @@ -718,6 +801,16 @@ void InitializeAsanInterceptors() { ASAN_INTERCEPT_FUNC(pthread_create); #endif ASAN_INTERCEPT_FUNC(pthread_join); + ASAN_INTERCEPT_FUNC(pthread_detach); + ASAN_INTERCEPT_FUNC(pthread_exit); +# endif + +# if ASAN_INTERCEPT_TIMEDJOIN + ASAN_INTERCEPT_FUNC(pthread_timedjoin_np); +#endif + +#if ASAN_INTERCEPT_TRYJOIN + ASAN_INTERCEPT_FUNC(pthread_tryjoin_np); #endif // Intercept atexit function. @@ -737,8 +830,6 @@ void InitializeAsanInterceptors() { ASAN_INTERCEPT_FUNC(vfork); #endif - InitializePlatformInterceptors(); - VReport(1, "AddressSanitizer: libc interceptors initialized\n"); } diff --git a/system/lib/compiler-rt/lib/asan/asan_interceptors.h b/system/lib/compiler-rt/lib/asan/asan_interceptors.h index c4bf087ea17f0..d00d05587b368 100644 --- a/system/lib/compiler-rt/lib/asan/asan_interceptors.h +++ b/system/lib/compiler-rt/lib/asan/asan_interceptors.h @@ -42,12 +42,10 @@ void InitializePlatformInterceptors(); // Use macro to describe if specific function should be // intercepted on a given platform. #if !SANITIZER_WINDOWS -# define ASAN_INTERCEPT_ATOLL_AND_STRTOLL 1 # define ASAN_INTERCEPT__LONGJMP 1 # define ASAN_INTERCEPT_INDEX 1 # define ASAN_INTERCEPT_PTHREAD_CREATE 1 #else -# define ASAN_INTERCEPT_ATOLL_AND_STRTOLL 0 # define ASAN_INTERCEPT__LONGJMP 0 # define ASAN_INTERCEPT_INDEX 0 # define ASAN_INTERCEPT_PTHREAD_CREATE 0 @@ -78,8 +76,8 @@ void InitializePlatformInterceptors(); # define ASAN_INTERCEPT___LONGJMP_CHK 0 #endif -#if ASAN_HAS_EXCEPTIONS && !SANITIZER_WINDOWS && !SANITIZER_SOLARIS && \ - !SANITIZER_NETBSD +#if ASAN_HAS_EXCEPTIONS && !SANITIZER_SOLARIS && !SANITIZER_NETBSD && \ + (!SANITIZER_WINDOWS || (defined(__MINGW32__) && defined(__i386__))) # define ASAN_INTERCEPT___CXA_THROW 1 # define ASAN_INTERCEPT___CXA_RETHROW_PRIMARY_EXCEPTION 1 # if defined(_GLIBCXX_SJLJ_EXCEPTIONS) || (SANITIZER_IOS && defined(__arm__)) @@ -112,6 +110,14 @@ void InitializePlatformInterceptors(); # define ASAN_INTERCEPT___STRDUP 0 #endif +#if SANITIZER_GLIBC && ASAN_INTERCEPT_PTHREAD_CREATE +# define ASAN_INTERCEPT_TIMEDJOIN 1 +# define ASAN_INTERCEPT_TRYJOIN 1 +#else +# define ASAN_INTERCEPT_TIMEDJOIN 0 +# define ASAN_INTERCEPT_TRYJOIN 0 +#endif + #if SANITIZER_LINUX && \ (defined(__arm__) || defined(__aarch64__) || defined(__i386__) || \ defined(__x86_64__) || SANITIZER_RISCV64 || SANITIZER_LOONGARCH64) @@ -158,6 +164,12 @@ DECLARE_REAL(char*, strstr, const char *s1, const char *s2) # define ASAN_INTERCEPT_FUNC(name) # endif // SANITIZER_APPLE +#define ASAN_INTERCEPTOR_ENTER(ctx, func) \ + AsanInterceptorContext _ctx = {#func}; \ + ctx = (void *)&_ctx; \ + (void) ctx; +#define COMMON_INTERCEPT_FUNCTION(name) ASAN_INTERCEPT_FUNC(name) + #endif // !SANITIZER_FUCHSIA #endif // ASAN_INTERCEPTORS_H diff --git a/system/lib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cpp b/system/lib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cpp index 9e275ba211e2f..7c12a0f5810ab 100644 --- a/system/lib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.cpp @@ -11,13 +11,54 @@ // ASan versions of memcpy, memmove, and memset. //===---------------------------------------------------------------------===// +#define SANITIZER_COMMON_NO_REDEFINE_BUILTINS + #include "asan_interceptors_memintrinsics.h" + +#include "asan_interceptors.h" #include "asan_report.h" #include "asan_stack.h" #include "asan_suppressions.h" using namespace __asan; +// memcpy is called during __asan_init() from the internals of printf(...). +// We do not treat memcpy with to==from as a bug. +// See http://llvm.org/bugs/show_bug.cgi?id=11763. +#define ASAN_MEMCPY_IMPL(ctx, to, from, size) \ + do { \ + if (LIKELY(replace_intrin_cached)) { \ + if (LIKELY(to != from)) { \ + CHECK_RANGES_OVERLAP("memcpy", to, size, from, size); \ + } \ + ASAN_READ_RANGE(ctx, from, size); \ + ASAN_WRITE_RANGE(ctx, to, size); \ + } else if (UNLIKELY(!asan_inited)) { \ + return internal_memcpy(to, from, size); \ + } \ + return REAL(memcpy)(to, from, size); \ + } while (0) + +// memset is called inside Printf. +#define ASAN_MEMSET_IMPL(ctx, block, c, size) \ + do { \ + if (LIKELY(replace_intrin_cached)) { \ + ASAN_WRITE_RANGE(ctx, block, size); \ + } else if (UNLIKELY(!asan_inited)) { \ + return internal_memset(block, c, size); \ + } \ + return REAL(memset)(block, c, size); \ + } while (0) + +#define ASAN_MEMMOVE_IMPL(ctx, to, from, size) \ + do { \ + if (LIKELY(replace_intrin_cached)) { \ + ASAN_READ_RANGE(ctx, from, size); \ + ASAN_WRITE_RANGE(ctx, to, size); \ + } \ + return internal_memmove(to, from, size); \ + } while (0) + void *__asan_memcpy(void *to, const void *from, uptr size) { ASAN_MEMCPY_IMPL(nullptr, to, from, size); } @@ -40,4 +81,26 @@ extern "C" decltype(__asan_memcpy) memcpy[[gnu::alias("__asan_memcpy")]]; extern "C" decltype(__asan_memmove) memmove[[gnu::alias("__asan_memmove")]]; extern "C" decltype(__asan_memset) memset[[gnu::alias("__asan_memset")]]; +#else // SANITIZER_FUCHSIA || SANITIZER_EMSCRIPTEN + +#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size) \ + do { \ + ASAN_INTERCEPTOR_ENTER(ctx, memmove); \ + ASAN_MEMMOVE_IMPL(ctx, to, from, size); \ + } while (false) + +#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size) \ + do { \ + ASAN_INTERCEPTOR_ENTER(ctx, memcpy); \ + ASAN_MEMCPY_IMPL(ctx, to, from, size); \ + } while (false) + +#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size) \ + do { \ + ASAN_INTERCEPTOR_ENTER(ctx, memset); \ + ASAN_MEMSET_IMPL(ctx, block, c, size); \ + } while (false) + +#include "sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc" + #endif // SANITIZER_FUCHSIA || SANITIZER_EMSCRIPTEN diff --git a/system/lib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h b/system/lib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h index bbc5390ceaa40..eb44f8f2f729b 100644 --- a/system/lib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h +++ b/system/lib/compiler-rt/lib/asan/asan_interceptors_memintrinsics.h @@ -79,43 +79,6 @@ struct AsanInterceptorContext { } \ } while (0) -// memcpy is called during __asan_init() from the internals of printf(...). -// We do not treat memcpy with to==from as a bug. -// See http://llvm.org/bugs/show_bug.cgi?id=11763. -#define ASAN_MEMCPY_IMPL(ctx, to, from, size) \ - do { \ - if (LIKELY(replace_intrin_cached)) { \ - if (LIKELY(to != from)) { \ - CHECK_RANGES_OVERLAP("memcpy", to, size, from, size); \ - } \ - ASAN_READ_RANGE(ctx, from, size); \ - ASAN_WRITE_RANGE(ctx, to, size); \ - } else if (UNLIKELY(!asan_inited)) { \ - return internal_memcpy(to, from, size); \ - } \ - return REAL(memcpy)(to, from, size); \ - } while (0) - -// memset is called inside Printf. -#define ASAN_MEMSET_IMPL(ctx, block, c, size) \ - do { \ - if (LIKELY(replace_intrin_cached)) { \ - ASAN_WRITE_RANGE(ctx, block, size); \ - } else if (UNLIKELY(!asan_inited)) { \ - return internal_memset(block, c, size); \ - } \ - return REAL(memset)(block, c, size); \ - } while (0) - -#define ASAN_MEMMOVE_IMPL(ctx, to, from, size) \ - do { \ - if (LIKELY(replace_intrin_cached)) { \ - ASAN_READ_RANGE(ctx, from, size); \ - ASAN_WRITE_RANGE(ctx, to, size); \ - } \ - return internal_memmove(to, from, size); \ - } while (0) - #define ASAN_READ_RANGE(ctx, offset, size) \ ACCESS_MEMORY_RANGE(ctx, offset, size, false) #define ASAN_WRITE_RANGE(ctx, offset, size) \ diff --git a/system/lib/compiler-rt/lib/asan/asan_interface_internal.h b/system/lib/compiler-rt/lib/asan/asan_interface_internal.h index 987f855c0f9ce..a998263780221 100644 --- a/system/lib/compiler-rt/lib/asan/asan_interface_internal.h +++ b/system/lib/compiler-rt/lib/asan/asan_interface_internal.h @@ -53,9 +53,10 @@ extern "C" { const char *module_name; // Module name as a C string. This pointer is a // unique identifier of a module. uptr has_dynamic_init; // Non-zero if the global has dynamic initializer. - uptr windows_padding; // TODO: Figure out how to remove this padding - // that's simply here to make the MSVC incremental - // linker happy... + __asan_global_source_location *gcc_location; // Source location of a global, + // used by GCC compiler. LLVM uses + // llvm-symbolizer that relies + // on DWARF debugging info. uptr odr_indicator; // The address of the ODR indicator symbol. }; diff --git a/system/lib/compiler-rt/lib/asan/asan_mac.cpp b/system/lib/compiler-rt/lib/asan/asan_mac.cpp index c9bd5fb8e1a8f..0c07500614478 100644 --- a/system/lib/compiler-rt/lib/asan/asan_mac.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_mac.cpp @@ -130,6 +130,18 @@ typedef void* dispatch_source_t; typedef u64 dispatch_time_t; typedef void (*dispatch_function_t)(void *block); typedef void* (*worker_t)(void *block); +typedef unsigned long dispatch_mach_reason; +typedef void *dispatch_mach_msg_t; +typedef int mach_error_t; +typedef void *dispatch_mach_t; + +typedef void (*dispatch_mach_handler_function_t)(void *context, + dispatch_mach_reason reason, + dispatch_mach_msg_t message, + mach_error_t error); +typedef void (^dispatch_mach_handler_t)(dispatch_mach_reason reason, + dispatch_mach_msg_t message, + mach_error_t error); // A wrapper for the ObjC blocks used to support libdispatch. typedef struct { @@ -160,7 +172,7 @@ void asan_dispatch_call_block_and_release(void *block) { VReport(2, "asan_dispatch_call_block_and_release(): " "context: %p, pthread_self: %p\n", - block, pthread_self()); + block, (void*)pthread_self()); asan_register_worker_thread(context->parent_tid, &stack); // Call the original dispatcher for the block. context->func(context->block); @@ -193,7 +205,7 @@ asan_block_context_t *alloc_asan_context(void *ctxt, dispatch_function_t func, asan_block_context_t *asan_ctxt = alloc_asan_context(ctxt, func, &stack); \ if (Verbosity() >= 2) { \ Report(#dispatch_x_f "(): context: %p, pthread_self: %p\n", \ - asan_ctxt, pthread_self()); \ + (void*)asan_ctxt, (void*)pthread_self()); \ PRINT_CURRENT_STACK(); \ } \ return REAL(dispatch_x_f)(dq, (void*)asan_ctxt, \ @@ -210,7 +222,7 @@ INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when, GET_STACK_TRACE_THREAD; asan_block_context_t *asan_ctxt = alloc_asan_context(ctxt, func, &stack); if (Verbosity() >= 2) { - Report("dispatch_after_f: %p\n", asan_ctxt); + Report("dispatch_after_f: %p\n", (void*)asan_ctxt); PRINT_CURRENT_STACK(); } return REAL(dispatch_after_f)(when, dq, (void*)asan_ctxt, @@ -224,7 +236,7 @@ INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group, asan_block_context_t *asan_ctxt = alloc_asan_context(ctxt, func, &stack); if (Verbosity() >= 2) { Report("dispatch_group_async_f(): context: %p, pthread_self: %p\n", - asan_ctxt, pthread_self()); + (void*)asan_ctxt, (void*)pthread_self()); PRINT_CURRENT_STACK(); } REAL(dispatch_group_async_f)(group, dq, (void*)asan_ctxt, @@ -241,6 +253,8 @@ void dispatch_after(dispatch_time_t when, dispatch_queue_t queue, void dispatch_source_set_cancel_handler(dispatch_source_t ds, void(^work)(void)); void dispatch_source_set_event_handler(dispatch_source_t ds, void(^work)(void)); +dispatch_mach_t dispatch_mach_create(const char *label, dispatch_queue_t queue, + dispatch_mach_handler_t handler); } #define GET_ASAN_BLOCK(work) \ @@ -290,6 +304,34 @@ INTERCEPTOR(void, dispatch_source_set_event_handler, GET_ASAN_BLOCK(work); REAL(dispatch_source_set_event_handler)(ds, asan_block); } + +INTERCEPTOR(void *, dispatch_mach_create, const char *label, + dispatch_queue_t dq, dispatch_mach_handler_t handler) { + int parent_tid = GetCurrentTidOrInvalid(); + return REAL(dispatch_mach_create)( + label, dq, + ^(dispatch_mach_reason reason, dispatch_mach_msg_t message, + mach_error_t error) { + GET_STACK_TRACE_THREAD; + asan_register_worker_thread(parent_tid, &stack); + handler(reason, message, error); + }); +} + +INTERCEPTOR(void *, dispatch_mach_create_f, const char *label, + dispatch_queue_t dq, void *ctxt, + dispatch_mach_handler_function_t handler) { + int parent_tid = GetCurrentTidOrInvalid(); + return REAL(dispatch_mach_create)( + label, dq, + ^(dispatch_mach_reason reason, dispatch_mach_msg_t message, + mach_error_t error) { + GET_STACK_TRACE_THREAD; + asan_register_worker_thread(parent_tid, &stack); + handler(ctxt, reason, message, error); + }); +} + #endif #endif // SANITIZER_APPLE diff --git a/system/lib/compiler-rt/lib/asan/asan_malloc_win.cpp b/system/lib/compiler-rt/lib/asan/asan_malloc_win.cpp index 4b76d4ebd3eb4..ff78d7646a90d 100644 --- a/system/lib/compiler-rt/lib/asan/asan_malloc_win.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_malloc_win.cpp @@ -508,10 +508,10 @@ void ReplaceSystemMalloc() { TryToOverrideFunction("_expand_base", (uptr)_expand); if (flags()->windows_hook_rtl_allocators) { - INTERCEPT_FUNCTION(HeapSize); - INTERCEPT_FUNCTION(HeapFree); - INTERCEPT_FUNCTION(HeapReAlloc); - INTERCEPT_FUNCTION(HeapAlloc); + ASAN_INTERCEPT_FUNC(HeapSize); + ASAN_INTERCEPT_FUNC(HeapFree); + ASAN_INTERCEPT_FUNC(HeapReAlloc); + ASAN_INTERCEPT_FUNC(HeapAlloc); // Undocumented functions must be intercepted by name, not by symbol. __interception::OverrideFunction("RtlSizeHeap", (uptr)WRAP(RtlSizeHeap), diff --git a/system/lib/compiler-rt/lib/asan/asan_memory_profile.cpp b/system/lib/compiler-rt/lib/asan/asan_memory_profile.cpp index 4fcd5600ed1a1..3396fc2bab94b 100644 --- a/system/lib/compiler-rt/lib/asan/asan_memory_profile.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_memory_profile.cpp @@ -11,12 +11,11 @@ // This file implements __sanitizer_print_memory_profile. //===----------------------------------------------------------------------===// +#include "asan/asan_allocator.h" +#include "lsan/lsan_common.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_stackdepot.h" #include "sanitizer_common/sanitizer_stacktrace.h" -#include "sanitizer_common/sanitizer_stoptheworld.h" -#include "lsan/lsan_common.h" -#include "asan/asan_allocator.h" #if CAN_SANITIZE_LEAKS @@ -100,17 +99,16 @@ static void ChunkCallback(uptr chunk, void *arg) { FindHeapChunkByAllocBeg(chunk)); } -static void MemoryProfileCB(const SuspendedThreadsList &suspended_threads_list, - void *argument) { +static void MemoryProfileCB(uptr top_percent, uptr max_number_of_contexts) { HeapProfile hp; + __lsan::LockAllocator(); __lsan::ForEachChunk(ChunkCallback, &hp); - uptr *Arg = reinterpret_cast(argument); - hp.Print(Arg[0], Arg[1]); + __lsan::UnlockAllocator(); + hp.Print(top_percent, max_number_of_contexts); if (Verbosity()) __asan_print_accumulated_stats(); } - } // namespace __asan #endif // CAN_SANITIZE_LEAKS @@ -120,10 +118,7 @@ SANITIZER_INTERFACE_ATTRIBUTE void __sanitizer_print_memory_profile(uptr top_percent, uptr max_number_of_contexts) { #if CAN_SANITIZE_LEAKS - uptr Arg[2]; - Arg[0] = top_percent; - Arg[1] = max_number_of_contexts; - __sanitizer::StopTheWorld(__asan::MemoryProfileCB, Arg); + __asan::MemoryProfileCB(top_percent, max_number_of_contexts); #endif // CAN_SANITIZE_LEAKS } } // extern "C" diff --git a/system/lib/compiler-rt/lib/asan/asan_poisoning.cpp b/system/lib/compiler-rt/lib/asan/asan_poisoning.cpp index e1ff872067ff9..4f9daf23ef7ca 100644 --- a/system/lib/compiler-rt/lib/asan/asan_poisoning.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_poisoning.cpp @@ -458,11 +458,11 @@ void __sanitizer_annotate_contiguous_container(const void *beg_p, // FIXME: Two of these three checks are disabled until we fix // https://github.com/google/sanitizers/issues/258. // if (d1 != d2) - // CHECK_EQ(*(u8*)MemToShadow(d1), old_mid - d1); + // DCHECK_EQ(*(u8*)MemToShadow(d1), old_mid - d1); if (a + granularity <= d1) - CHECK_EQ(*(u8 *)MemToShadow(a), 0); + DCHECK_EQ(*(u8 *)MemToShadow(a), 0); // if (d2 + granularity <= c && c <= end) - // CHECK_EQ(*(u8 *)MemToShadow(c - granularity), + // DCHECK_EQ(*(u8 *)MemToShadow(c - granularity), // kAsanContiguousContainerOOBMagic); uptr b1 = RoundDownTo(new_end, granularity); diff --git a/system/lib/compiler-rt/lib/asan/asan_stack.h b/system/lib/compiler-rt/lib/asan/asan_stack.h index b9575d2f427ee..02a76af847ae6 100644 --- a/system/lib/compiler-rt/lib/asan/asan_stack.h +++ b/system/lib/compiler-rt/lib/asan/asan_stack.h @@ -32,24 +32,24 @@ u32 GetMallocContextSize(); // as early as possible (in functions exposed to the user), as we generally // don't want stack trace to contain functions from ASan internals. -#define GET_STACK_TRACE(max_size, fast) \ - BufferedStackTrace stack; \ - if (max_size <= 2) { \ - stack.size = max_size; \ - if (max_size > 0) { \ - stack.top_frame_bp = GET_CURRENT_FRAME(); \ - stack.trace_buffer[0] = StackTrace::GetCurrentPc(); \ - if (max_size > 1) stack.trace_buffer[1] = GET_CALLER_PC(); \ - } \ - } else { \ - stack.Unwind(StackTrace::GetCurrentPc(), \ - GET_CURRENT_FRAME(), nullptr, fast, max_size); \ +#define GET_STACK_TRACE(max_size, fast) \ + UNINITIALIZED BufferedStackTrace stack; \ + if (max_size <= 2) { \ + stack.size = max_size; \ + if (max_size > 0) { \ + stack.top_frame_bp = GET_CURRENT_FRAME(); \ + stack.trace_buffer[0] = StackTrace::GetCurrentPc(); \ + if (max_size > 1) \ + stack.trace_buffer[1] = GET_CALLER_PC(); \ + } \ + } else { \ + stack.Unwind(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME(), nullptr, \ + fast, max_size); \ } -#define GET_STACK_TRACE_FATAL(pc, bp) \ - BufferedStackTrace stack; \ - stack.Unwind(pc, bp, nullptr, \ - common_flags()->fast_unwind_on_fatal) +#define GET_STACK_TRACE_FATAL(pc, bp) \ + UNINITIALIZED BufferedStackTrace stack; \ + stack.Unwind(pc, bp, nullptr, common_flags()->fast_unwind_on_fatal) #define GET_STACK_TRACE_FATAL_HERE \ GET_STACK_TRACE(kStackTraceMax, common_flags()->fast_unwind_on_fatal) diff --git a/system/lib/compiler-rt/lib/asan/asan_thread.cpp b/system/lib/compiler-rt/lib/asan/asan_thread.cpp index c991c085a3fd9..df5ef82036c68 100644 --- a/system/lib/compiler-rt/lib/asan/asan_thread.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_thread.cpp @@ -10,24 +10,25 @@ // // Thread-related code. //===----------------------------------------------------------------------===// +#include "asan_thread.h" + #include "asan_allocator.h" #include "asan_interceptors.h" +#include "asan_mapping.h" #include "asan_poisoning.h" #include "asan_stack.h" -#include "asan_thread.h" -#include "asan_mapping.h" +#include "lsan/lsan_common.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_placement_new.h" #include "sanitizer_common/sanitizer_stackdepot.h" #include "sanitizer_common/sanitizer_tls_get_addr.h" -#include "lsan/lsan_common.h" namespace __asan { // AsanThreadContext implementation. void AsanThreadContext::OnCreated(void *arg) { - CreateThreadContextArgs *args = static_cast(arg); + CreateThreadContextArgs *args = static_cast(arg); if (args->stack) stack_id = StackDepotPut(*args->stack); thread = args->thread; @@ -39,34 +40,50 @@ void AsanThreadContext::OnFinished() { thread = nullptr; } -// MIPS requires aligned address -static ALIGNED(16) char thread_registry_placeholder[sizeof(ThreadRegistry)]; static ThreadRegistry *asan_thread_registry; +static ThreadArgRetval *thread_data; static Mutex mu_for_thread_context; static LowLevelAllocator allocator_for_thread_context; static ThreadContextBase *GetAsanThreadContext(u32 tid) { Lock lock(&mu_for_thread_context); - return new(allocator_for_thread_context) AsanThreadContext(tid); + return new (allocator_for_thread_context) AsanThreadContext(tid); } -ThreadRegistry &asanThreadRegistry() { +static void InitThreads() { static bool initialized; // Don't worry about thread_safety - this should be called when there is // a single thread. - if (!initialized) { - // Never reuse ASan threads: we store pointer to AsanThreadContext - // in TSD and can't reliably tell when no more TSD destructors will - // be called. It would be wrong to reuse AsanThreadContext for another - // thread before all TSD destructors will be called for it. - asan_thread_registry = - new (thread_registry_placeholder) ThreadRegistry(GetAsanThreadContext); - initialized = true; - } + if (LIKELY(initialized)) + return; + // Never reuse ASan threads: we store pointer to AsanThreadContext + // in TSD and can't reliably tell when no more TSD destructors will + // be called. It would be wrong to reuse AsanThreadContext for another + // thread before all TSD destructors will be called for it. + + // MIPS requires aligned address + static ALIGNED(alignof( + ThreadRegistry)) char thread_registry_placeholder[sizeof(ThreadRegistry)]; + static ALIGNED(alignof( + ThreadArgRetval)) char thread_data_placeholder[sizeof(ThreadArgRetval)]; + + asan_thread_registry = + new (thread_registry_placeholder) ThreadRegistry(GetAsanThreadContext); + thread_data = new (thread_data_placeholder) ThreadArgRetval(); + initialized = true; +} + +ThreadRegistry &asanThreadRegistry() { + InitThreads(); return *asan_thread_registry; } +ThreadArgRetval &asanThreadArgRetval() { + InitThreads(); + return *thread_data; +} + AsanThreadContext *GetThreadContextByTidLocked(u32 tid) { return static_cast( asanThreadRegistry().GetThreadLocked(tid)); @@ -79,7 +96,7 @@ AsanThread *AsanThread::Create(thread_callback_t start_routine, void *arg, bool detached) { uptr PageSize = GetPageSizeCached(); uptr size = RoundUpTo(sizeof(AsanThread), PageSize); - AsanThread *thread = (AsanThread*)MmapOrDie(size, __func__); + AsanThread *thread = (AsanThread *)MmapOrDie(size, __func__); thread->start_routine_ = start_routine; thread->arg_ = arg; AsanThreadContext::CreateThreadContextArgs args = {thread, stack}; @@ -89,7 +106,7 @@ AsanThread *AsanThread::Create(thread_callback_t start_routine, void *arg, } void AsanThread::TSDDtor(void *tsd) { - AsanThreadContext *context = (AsanThreadContext*)tsd; + AsanThreadContext *context = (AsanThreadContext *)tsd; VReport(1, "T%d TSDDtor\n", context->tid); if (context->thread) context->thread->Destroy(); @@ -145,8 +162,7 @@ void AsanThread::StartSwitchFiber(FakeStack **fake_stack_save, uptr bottom, current_fake_stack->Destroy(this->tid()); } -void AsanThread::FinishSwitchFiber(FakeStack *fake_stack_save, - uptr *bottom_old, +void AsanThread::FinishSwitchFiber(FakeStack *fake_stack_save, uptr *bottom_old, uptr *size_old) { if (!atomic_load(&stack_switching_, memory_order_relaxed)) { Report("ERROR: finishing a fiber switch that has not started\n"); @@ -172,7 +188,8 @@ void AsanThread::FinishSwitchFiber(FakeStack *fake_stack_save, inline AsanThread::StackBounds AsanThread::GetStackBounds() const { if (!atomic_load(&stack_switching_, memory_order_acquire)) { // Make sure the stack bounds are fully initialized. - if (stack_bottom_ >= stack_top_) return {0, 0}; + if (stack_bottom_ >= stack_top_) + return {0, 0}; return {stack_bottom_, stack_top_}; } char local; @@ -185,13 +202,9 @@ inline AsanThread::StackBounds AsanThread::GetStackBounds() const { return {stack_bottom_, stack_top_}; } -uptr AsanThread::stack_top() { - return GetStackBounds().top; -} +uptr AsanThread::stack_top() { return GetStackBounds().top; } -uptr AsanThread::stack_bottom() { - return GetStackBounds().bottom; -} +uptr AsanThread::stack_bottom() { return GetStackBounds().bottom; } uptr AsanThread::stack_size() { const auto bounds = GetStackBounds(); @@ -212,8 +225,8 @@ FakeStack *AsanThread::AsyncSignalSafeLazyInitFakeStack() { // This CAS checks if the state was 0 and if so changes it to state 1, // if that was successful, it initializes the pointer. if (atomic_compare_exchange_strong( - reinterpret_cast(&fake_stack_), &old_val, 1UL, - memory_order_relaxed)) { + reinterpret_cast(&fake_stack_), &old_val, 1UL, + memory_order_relaxed)) { uptr stack_size_log = Log2(RoundUpToPowerOfTwo(stack_size)); CHECK_LE(flags()->min_uar_stack_size_log, flags()->max_uar_stack_size_log); stack_size_log = @@ -267,7 +280,8 @@ thread_return_t AsanThread::ThreadStart(tid_t os_id) { asanThreadRegistry().StartThread(tid(), os_id, ThreadType::Regular, nullptr); #if !SANITIZER_EMSCRIPTEN - if (common_flags()->use_sigaltstack) SetAlternateSignalStack(); + if (common_flags()->use_sigaltstack) + SetAlternateSignalStack(); #endif if (!start_routine_) { @@ -310,6 +324,7 @@ void AsanThread::SetThreadStackAndTls(const InitOptions *options) { GetThreadStackAndTls(tid() == kMainTid, &stack_bottom_, &stack_size, &tls_begin_, &tls_size); stack_top_ = RoundDownTo(stack_bottom_ + stack_size, ASAN_SHADOW_GRANULARITY); + stack_bottom_ = RoundDownTo(stack_bottom_, ASAN_SHADOW_GRANULARITY); tls_end_ = tls_begin_ + tls_size; dtls_ = DTLS_Get(); @@ -343,14 +358,14 @@ bool AsanThread::GetStackFrameAccessByAddr(uptr addr, bottom = fake_stack->AddrIsInFakeStack(addr); CHECK(bottom); access->offset = addr - bottom; - access->frame_pc = ((uptr*)bottom)[2]; - access->frame_descr = (const char *)((uptr*)bottom)[1]; + access->frame_pc = ((uptr *)bottom)[2]; + access->frame_descr = (const char *)((uptr *)bottom)[1]; return true; } uptr aligned_addr = RoundDownTo(addr, SANITIZER_WORDSIZE / 8); // align addr. uptr mem_ptr = RoundDownTo(aligned_addr, ASAN_SHADOW_GRANULARITY); - u8 *shadow_ptr = (u8*)MemToShadow(aligned_addr); - u8 *shadow_bottom = (u8*)MemToShadow(bottom); + u8 *shadow_ptr = (u8 *)MemToShadow(aligned_addr); + u8 *shadow_bottom = (u8 *)MemToShadow(bottom); while (shadow_ptr >= shadow_bottom && *shadow_ptr != kAsanStackLeftRedzoneMagic) { @@ -372,7 +387,7 @@ bool AsanThread::GetStackFrameAccessByAddr(uptr addr, CHECK(ptr[0] == kCurrentStackFrameMagic); access->offset = addr - (uptr)ptr; access->frame_pc = ptr[2]; - access->frame_descr = (const char*)ptr[1]; + access->frame_descr = (const char *)ptr[1]; return true; } @@ -390,8 +405,8 @@ uptr AsanThread::GetStackVariableShadowStart(uptr addr) { } uptr aligned_addr = RoundDownTo(addr, SANITIZER_WORDSIZE / 8); // align addr. - u8 *shadow_ptr = (u8*)MemToShadow(aligned_addr); - u8 *shadow_bottom = (u8*)MemToShadow(bottom); + u8 *shadow_ptr = (u8 *)MemToShadow(aligned_addr); + u8 *shadow_bottom = (u8 *)MemToShadow(bottom); while (shadow_ptr >= shadow_bottom && (*shadow_ptr != kAsanStackLeftRedzoneMagic && @@ -475,16 +490,23 @@ void EnsureMainThreadIDIsCorrect() { __asan::AsanThread *GetAsanThreadByOsIDLocked(tid_t os_id) { __asan::AsanThreadContext *context = static_cast<__asan::AsanThreadContext *>( __asan::asanThreadRegistry().FindThreadContextByOsIDLocked(os_id)); - if (!context) return nullptr; + if (!context) + return nullptr; return context->thread; } -} // namespace __asan +} // namespace __asan // --- Implementation of LSan-specific functions --- {{{1 namespace __lsan { -void LockThreadRegistry() { __asan::asanThreadRegistry().Lock(); } +void LockThreads() { + __asan::asanThreadRegistry().Lock(); + __asan::asanThreadArgRetval().Lock(); +} -void UnlockThreadRegistry() { __asan::asanThreadRegistry().Unlock(); } +void UnlockThreads() { + __asan::asanThreadArgRetval().Unlock(); + __asan::asanThreadRegistry().Unlock(); +} static ThreadRegistry *GetAsanThreadRegistryLocked() { __asan::asanThreadRegistry().CheckLocked(); @@ -497,7 +519,8 @@ bool GetThreadRangesLocked(tid_t os_id, uptr *stack_begin, uptr *stack_end, uptr *tls_begin, uptr *tls_end, uptr *cache_begin, uptr *cache_end, DTLS **dtls) { __asan::AsanThread *t = __asan::GetAsanThreadByOsIDLocked(os_id); - if (!t) return false; + if (!t) + return false; *stack_begin = t->stack_bottom(); *stack_end = t->stack_top(); *tls_begin = t->tls_begin(); @@ -538,33 +561,7 @@ void GetThreadExtraStackRangesLocked(InternalMmapVector *ranges) { } void GetAdditionalThreadContextPtrsLocked(InternalMmapVector *ptrs) { - GetAsanThreadRegistryLocked()->RunCallbackForEachThreadLocked( - [](ThreadContextBase *tctx, void *ptrs) { - // Look for the arg pointer of threads that have been created or are - // running. This is necessary to prevent false positive leaks due to the - // AsanThread holding the only live reference to a heap object. This - // can happen because the `pthread_create()` interceptor doesn't wait - // for the child thread to start before returning and thus loosing the - // the only live reference to the heap object on the stack. - - __asan::AsanThreadContext *atctx = - static_cast<__asan::AsanThreadContext *>(tctx); - - // Note ThreadStatusRunning is required because there is a small window - // where the thread status switches to `ThreadStatusRunning` but the - // `arg` pointer still isn't on the stack yet. - if (atctx->status != ThreadStatusCreated && - atctx->status != ThreadStatusRunning) - return; - - uptr thread_arg = reinterpret_cast(atctx->thread->get_arg()); - if (!thread_arg) - return; - - auto ptrsVec = reinterpret_cast *>(ptrs); - ptrsVec->push_back(thread_arg); - }, - ptrs); + __asan::asanThreadArgRetval().GetAllPtrsLocked(ptrs); } void GetRunningThreadsLocked(InternalMmapVector *threads) { @@ -577,11 +574,7 @@ void GetRunningThreadsLocked(InternalMmapVector *threads) { threads); } -void FinishThreadLocked(u32 tid) { - GetAsanThreadRegistryLocked()->FinishThread(tid); -} - -} // namespace __lsan +} // namespace __lsan namespace __sanitizer { ThreadRegistry *GetThreadRegistryLocked() { @@ -601,20 +594,18 @@ void __sanitizer_start_switch_fiber(void **fakestacksave, const void *bottom, VReport(1, "__asan_start_switch_fiber called from unknown thread\n"); return; } - t->StartSwitchFiber((FakeStack**)fakestacksave, (uptr)bottom, size); + t->StartSwitchFiber((FakeStack **)fakestacksave, (uptr)bottom, size); } SANITIZER_INTERFACE_ATTRIBUTE -void __sanitizer_finish_switch_fiber(void* fakestack, - const void **bottom_old, +void __sanitizer_finish_switch_fiber(void *fakestack, const void **bottom_old, uptr *size_old) { AsanThread *t = GetCurrentThread(); if (!t) { VReport(1, "__asan_finish_switch_fiber called from unknown thread\n"); return; } - t->FinishSwitchFiber((FakeStack*)fakestack, - (uptr*)bottom_old, - (uptr*)size_old); + t->FinishSwitchFiber((FakeStack *)fakestack, (uptr *)bottom_old, + (uptr *)size_old); } } diff --git a/system/lib/compiler-rt/lib/asan/asan_thread.h b/system/lib/compiler-rt/lib/asan/asan_thread.h index 801a3960ec6cb..c131dd40d8647 100644 --- a/system/lib/compiler-rt/lib/asan/asan_thread.h +++ b/system/lib/compiler-rt/lib/asan/asan_thread.h @@ -15,11 +15,12 @@ #define ASAN_THREAD_H #include "asan_allocator.h" -#include "asan_internal.h" #include "asan_fake_stack.h" +#include "asan_internal.h" #include "asan_stats.h" #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_libc.h" +#include "sanitizer_common/sanitizer_thread_arg_retval.h" #include "sanitizer_common/sanitizer_thread_registry.h" namespace __sanitizer { @@ -129,7 +130,7 @@ class AsanThread { void *extra_spill_area() { return &extra_spill_area_; } - void *get_arg() { return arg_; } + void *get_arg() const { return arg_; } private: // NOTE: There is no AsanThread constructor. It is allocated @@ -171,6 +172,7 @@ class AsanThread { // Returns a single instance of registry. ThreadRegistry &asanThreadRegistry(); +ThreadArgRetval &asanThreadArgRetval(); // Must be called under ThreadRegistryLock. AsanThreadContext *GetThreadContextByTidLocked(u32 tid); diff --git a/system/lib/compiler-rt/lib/asan/asan_win.cpp b/system/lib/compiler-rt/lib/asan/asan_win.cpp index 7dbd7ab98a17b..25f2e6cd551fe 100644 --- a/system/lib/compiler-rt/lib/asan/asan_win.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_win.cpp @@ -159,6 +159,8 @@ INTERCEPTOR_WINAPI(HANDLE, CreateThread, LPSECURITY_ATTRIBUTES security, namespace __asan { void InitializePlatformInterceptors() { + __interception::SetErrorReportCallback(Report); + // The interceptors were not designed to be removable, so we have to keep this // module alive for the life of the process. HMODULE pinned; @@ -194,9 +196,12 @@ void AsanApplyToGlobals(globals_op_fptr op, const void *needle) { } void FlushUnneededASanShadowMemory(uptr p, uptr size) { + // Only asan on 64-bit Windows supports committing shadow memory on demand. +#if SANITIZER_WINDOWS64 // Since asan's mapping is compacting, the shadow chunk may be // not page-aligned, so we only flush the page-aligned portion. ReleaseMemoryPagesToOS(MemToShadow(p), MemToShadow(p + size)); +#endif } // ---------------------- TSD ---------------- {{{ diff --git a/system/lib/compiler-rt/lib/asan/asan_win_dll_thunk.cpp b/system/lib/compiler-rt/lib/asan/asan_win_dll_thunk.cpp index e3a90f18ed81a..0fa636bec0d00 100644 --- a/system/lib/compiler-rt/lib/asan/asan_win_dll_thunk.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_win_dll_thunk.cpp @@ -65,6 +65,7 @@ INTERCEPT_WRAP_W_W(_expand_dbg) INTERCEPT_LIBRARY_FUNCTION(atoi); INTERCEPT_LIBRARY_FUNCTION(atol); +INTERCEPT_LIBRARY_FUNCTION(atoll); INTERCEPT_LIBRARY_FUNCTION(frexp); INTERCEPT_LIBRARY_FUNCTION(longjmp); #if SANITIZER_INTERCEPT_MEMCHR @@ -91,6 +92,7 @@ INTERCEPT_LIBRARY_FUNCTION(strspn); INTERCEPT_LIBRARY_FUNCTION(strstr); INTERCEPT_LIBRARY_FUNCTION(strtok); INTERCEPT_LIBRARY_FUNCTION(strtol); +INTERCEPT_LIBRARY_FUNCTION(strtoll); INTERCEPT_LIBRARY_FUNCTION(wcslen); INTERCEPT_LIBRARY_FUNCTION(wcsnlen); diff --git a/system/lib/compiler-rt/lib/builtins/absvdi2.c b/system/lib/compiler-rt/lib/builtins/absvdi2.c index b9566cd874fe4..291ab5f7f91d5 100644 --- a/system/lib/compiler-rt/lib/builtins/absvdi2.c +++ b/system/lib/compiler-rt/lib/builtins/absvdi2.c @@ -18,7 +18,7 @@ COMPILER_RT_ABI di_int __absvdi2(di_int a) { const int N = (int)(sizeof(di_int) * CHAR_BIT); - if (a == ((di_int)1 << (N - 1))) + if (a == ((di_int)((du_int)1 << (N - 1)))) compilerrt_abort(); const di_int t = a >> (N - 1); return (a ^ t) - t; diff --git a/system/lib/compiler-rt/lib/builtins/absvsi2.c b/system/lib/compiler-rt/lib/builtins/absvsi2.c index 9d5de7e8a3f22..9977c33d8f7ea 100644 --- a/system/lib/compiler-rt/lib/builtins/absvsi2.c +++ b/system/lib/compiler-rt/lib/builtins/absvsi2.c @@ -18,7 +18,7 @@ COMPILER_RT_ABI si_int __absvsi2(si_int a) { const int N = (int)(sizeof(si_int) * CHAR_BIT); - if (a == ((si_int)1 << (N - 1))) + if (a == ((si_int)((su_int)1 << (N - 1)))) compilerrt_abort(); const si_int t = a >> (N - 1); return (a ^ t) - t; diff --git a/system/lib/compiler-rt/lib/builtins/addtf3.c b/system/lib/compiler-rt/lib/builtins/addtf3.c index 86e4f4cfc3fc6..2cb3a4d591917 100644 --- a/system/lib/compiler-rt/lib/builtins/addtf3.c +++ b/system/lib/compiler-rt/lib/builtins/addtf3.c @@ -13,7 +13,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) #include "fp_add_impl.inc" COMPILER_RT_ABI fp_t __addtf3(fp_t a, fp_t b) { diff --git a/system/lib/compiler-rt/lib/builtins/ashlti3.c b/system/lib/compiler-rt/lib/builtins/ashlti3.c index 2d7bd4a893809..99a133ffa22f8 100644 --- a/system/lib/compiler-rt/lib/builtins/ashlti3.c +++ b/system/lib/compiler-rt/lib/builtins/ashlti3.c @@ -18,7 +18,7 @@ // Precondition: 0 <= b < bits_in_tword -COMPILER_RT_ABI ti_int __ashlti3(ti_int a, si_int b) { +COMPILER_RT_ABI ti_int __ashlti3(ti_int a, int b) { const int bits_in_dword = (int)(sizeof(di_int) * CHAR_BIT); twords input; twords result; diff --git a/system/lib/compiler-rt/lib/builtins/ashrti3.c b/system/lib/compiler-rt/lib/builtins/ashrti3.c index f573b6d6ccba4..b306051df0287 100644 --- a/system/lib/compiler-rt/lib/builtins/ashrti3.c +++ b/system/lib/compiler-rt/lib/builtins/ashrti3.c @@ -18,7 +18,7 @@ // Precondition: 0 <= b < bits_in_tword -COMPILER_RT_ABI ti_int __ashrti3(ti_int a, si_int b) { +COMPILER_RT_ABI ti_int __ashrti3(ti_int a, int b) { const int bits_in_dword = (int)(sizeof(di_int) * CHAR_BIT); twords input; twords result; diff --git a/system/lib/compiler-rt/lib/builtins/assembly.h b/system/lib/compiler-rt/lib/builtins/assembly.h index 69a3d8620f924..169d49683f503 100644 --- a/system/lib/compiler-rt/lib/builtins/assembly.h +++ b/system/lib/compiler-rt/lib/builtins/assembly.h @@ -267,7 +267,7 @@ #define DEFINE_COMPILERRT_FUNCTION_ALIAS(name, target) \ .globl SYMBOL_NAME(name) SEPARATOR \ SYMBOL_IS_FUNC(SYMBOL_NAME(name)) SEPARATOR \ - DECLARE_SYMBOL_VISIBILITY(SYMBOL_NAME(name)) SEPARATOR \ + DECLARE_SYMBOL_VISIBILITY(name) SEPARATOR \ .set SYMBOL_NAME(name), SYMBOL_NAME(target) SEPARATOR #if defined(__ARM_EABI__) diff --git a/system/lib/compiler-rt/lib/builtins/clear_cache.c b/system/lib/compiler-rt/lib/builtins/clear_cache.c index 8993761eb3d42..2ac99b25c243f 100644 --- a/system/lib/compiler-rt/lib/builtins/clear_cache.c +++ b/system/lib/compiler-rt/lib/builtins/clear_cache.c @@ -110,10 +110,14 @@ void __clear_cache(void *start, void *end) { "jr.hb $at\n" "move $at, $0\n" ".set at"); -#else +#elif defined(__linux__) || defined(__OpenBSD__) // Pre-R6 may not be globalized. And some implementations may give strange // synci_step. So, let's use libc call for it. - cacheflush(start, end_int - start_int, BCACHE); + _flush_cache(start, end_int - start_int, BCACHE); +#else + (void)start_int; + (void)end_int; + compilerrt_abort(); #endif } #elif defined(__aarch64__) && !defined(__APPLE__) diff --git a/system/lib/compiler-rt/lib/builtins/comparetf2.c b/system/lib/compiler-rt/lib/builtins/comparetf2.c index f1592454138cd..be5e9e5e44dd2 100644 --- a/system/lib/compiler-rt/lib/builtins/comparetf2.c +++ b/system/lib/compiler-rt/lib/builtins/comparetf2.c @@ -39,7 +39,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) #include "fp_compare_impl.inc" COMPILER_RT_ABI CMP_RESULT __letf2(fp_t a, fp_t b) { return __leXf2__(a, b); } diff --git a/system/lib/compiler-rt/lib/builtins/cpu_model.c b/system/lib/compiler-rt/lib/builtins/cpu_model.c index f5ad530c7e886..0bd7296fb2526 100644 --- a/system/lib/compiler-rt/lib/builtins/cpu_model.c +++ b/system/lib/compiler-rt/lib/builtins/cpu_model.c @@ -113,6 +113,7 @@ enum ProcessorSubtypes { ZHAOXIN_FAM7H_LUJIAZUI, AMDFAM19H_ZNVER4, INTEL_COREI7_GRANITERAPIDS, + INTEL_COREI7_GRANITERAPIDS_D, CPU_SUBTYPE_MAX }; @@ -474,13 +475,19 @@ getIntelProcessorTypeAndSubtype(unsigned Family, unsigned Model, break; // Granite Rapids: - case 0xae: case 0xad: CPU = "graniterapids"; *Type = INTEL_COREI7; *Subtype = INTEL_COREI7_GRANITERAPIDS; break; + // Granite Rapids D: + case 0xae: + CPU = "graniterapids-d"; + *Type = INTEL_COREI7; + *Subtype = INTEL_COREI7_GRANITERAPIDS_D; + break; + case 0x1c: // Most 45 nm Intel Atom processors case 0x26: // 45 nm Atom Lincroft case 0x27: // 32 nm Atom Medfield @@ -646,7 +653,7 @@ getAMDProcessorTypeAndSubtype(unsigned Family, unsigned Model, static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, unsigned *Features) { - unsigned EAX, EBX; + unsigned EAX = 0, EBX = 0; #define setFeature(F) \ Features[F / 32] |= 1U << (F % 32) @@ -744,8 +751,11 @@ static void getAvailableFeatures(unsigned ECX, unsigned EDX, unsigned MaxLeaf, if (HasLeaf7 && ((EDX >> 8) & 1) && HasAVX512Save) setFeature(FEATURE_AVX512VP2INTERSECT); + // EAX from subleaf 0 is the maximum subleaf supported. Some CPUs don't + // return all 0s for invalid subleaves so check the limit. bool HasLeaf7Subleaf1 = - MaxLeaf >= 0x7 && !getX86CpuIDAndInfoEx(0x7, 0x1, &EAX, &EBX, &ECX, &EDX); + HasLeaf7 && EAX >= 1 && + !getX86CpuIDAndInfoEx(0x7, 0x1, &EAX, &EBX, &ECX, &EDX); if (HasLeaf7Subleaf1 && ((EAX >> 5) & 1) && HasAVX512Save) setFeature(FEATURE_AVX512BF16); @@ -839,6 +849,39 @@ int CONSTRUCTOR_ATTRIBUTE __cpu_indicator_init(void) { } #elif defined(__aarch64__) +// LSE support detection for out-of-line atomics +// using HWCAP and Auxiliary vector +_Bool __aarch64_have_lse_atomics + __attribute__((visibility("hidden"), nocommon)); + +#if defined(__has_include) +#if __has_include() +#include + +#if __has_include() +#include +#else +typedef struct __ifunc_arg_t { + unsigned long _size; + unsigned long _hwcap; + unsigned long _hwcap2; +} __ifunc_arg_t; +#endif // __has_include() + +#if __has_include() +#include + +#if defined(__ANDROID__) +#include +#include +#elif defined(__Fuchsia__) +#include +#include +#endif + +#ifndef _IFUNC_ARG_HWCAP +#define _IFUNC_ARG_HWCAP (1ULL << 62) +#endif #ifndef AT_HWCAP #define AT_HWCAP 16 #endif @@ -1009,25 +1052,6 @@ int CONSTRUCTOR_ATTRIBUTE __cpu_indicator_init(void) { #define HWCAP2_SVE_EBF16 (1UL << 33) #endif -// LSE support detection for out-of-line atomics -// using HWCAP and Auxiliary vector -_Bool __aarch64_have_lse_atomics - __attribute__((visibility("hidden"), nocommon)); - -#if defined(__has_include) -#if __has_include() -#include -#if __has_include() -#include - -#if defined(__ANDROID__) -#include -#include -#elif defined(__Fuchsia__) -#include -#include -#endif - // Detect Exynos 9810 CPU #define IF_EXYNOS9810 \ char arch[PROP_VALUE_MAX]; \ @@ -1140,11 +1164,16 @@ struct { // As features grows new fields could be added } __aarch64_cpu_features __attribute__((visibility("hidden"), nocommon)); -void init_cpu_features_resolver(unsigned long hwcap, unsigned long hwcap2) { +void init_cpu_features_resolver(unsigned long hwcap, const __ifunc_arg_t *arg) { #define setCPUFeature(F) __aarch64_cpu_features.features |= 1ULL << F #define getCPUFeature(id, ftr) __asm__("mrs %0, " #id : "=r"(ftr)) #define extractBits(val, start, number) \ (val & ((1ULL << number) - 1ULL) << start) >> start + if (__aarch64_cpu_features.features) + return; + unsigned long hwcap2 = 0; + if (hwcap & _IFUNC_ARG_HWCAP) + hwcap2 = arg->_hwcap2; if (hwcap & HWCAP_CRC32) setCPUFeature(FEAT_CRC); if (hwcap & HWCAP_PMULL) @@ -1320,6 +1349,7 @@ void init_cpu_features_resolver(unsigned long hwcap, unsigned long hwcap2) { if (hwcap & HWCAP_SHA3) setCPUFeature(FEAT_SHA3); } + setCPUFeature(FEAT_MAX); } void CONSTRUCTOR_ATTRIBUTE init_cpu_features(void) { @@ -1328,7 +1358,6 @@ void CONSTRUCTOR_ATTRIBUTE init_cpu_features(void) { // CPU features already initialized. if (__aarch64_cpu_features.features) return; - setCPUFeature(FEAT_MAX); #if defined(__FreeBSD__) int res = 0; res = elf_aux_info(AT_HWCAP, &hwcap, sizeof hwcap); @@ -1344,7 +1373,11 @@ void CONSTRUCTOR_ATTRIBUTE init_cpu_features(void) { hwcap = getauxval(AT_HWCAP); hwcap2 = getauxval(AT_HWCAP2); #endif // defined(__FreeBSD__) - init_cpu_features_resolver(hwcap, hwcap2); + __ifunc_arg_t arg; + arg._size = sizeof(__ifunc_arg_t); + arg._hwcap = hwcap; + arg._hwcap2 = hwcap2; + init_cpu_features_resolver(hwcap | _IFUNC_ARG_HWCAP, &arg); #undef extractBits #undef getCPUFeature #undef setCPUFeature diff --git a/system/lib/compiler-rt/lib/builtins/crtbegin.c b/system/lib/compiler-rt/lib/builtins/crtbegin.c new file mode 100644 index 0000000000000..a0860ca12ea03 --- /dev/null +++ b/system/lib/compiler-rt/lib/builtins/crtbegin.c @@ -0,0 +1,135 @@ +//===-- crtbegin.c - Start of constructors and destructors ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include + +__attribute__((visibility("hidden"))) void *__dso_handle = &__dso_handle; + +#ifdef EH_USE_FRAME_REGISTRY +__extension__ static void *__EH_FRAME_LIST__[] + __attribute__((section(".eh_frame"), aligned(sizeof(void *)))) = {}; + +extern void __register_frame_info(const void *, void *) __attribute__((weak)); +extern void *__deregister_frame_info(const void *) __attribute__((weak)); +#endif + +#ifndef CRT_HAS_INITFINI_ARRAY +typedef void (*fp)(void); + +static fp __CTOR_LIST__[] + __attribute__((section(".ctors"), aligned(sizeof(fp)))) = {(fp)-1}; +extern fp __CTOR_LIST_END__[]; +#endif + +extern void __cxa_finalize(void *) __attribute__((weak)); + +static void __attribute__((used)) __do_init(void) { + static _Bool __initialized; + if (__builtin_expect(__initialized, 0)) + return; + __initialized = 1; + +#ifdef EH_USE_FRAME_REGISTRY + static struct { void *p[8]; } __object; + if (__register_frame_info) + __register_frame_info(__EH_FRAME_LIST__, &__object); +#endif +#ifndef CRT_HAS_INITFINI_ARRAY + const size_t n = __CTOR_LIST_END__ - __CTOR_LIST__ - 1; + for (size_t i = n; i >= 1; i--) __CTOR_LIST__[i](); +#endif +} + +#ifdef CRT_HAS_INITFINI_ARRAY +__attribute__((section(".init_array"), + used)) static void (*__init)(void) = __do_init; +#elif defined(__i386__) || defined(__x86_64__) +__asm__(".pushsection .init,\"ax\",@progbits\n\t" + "call __do_init\n\t" + ".popsection"); +#elif defined(__riscv) +__asm__(".pushsection .init,\"ax\",%progbits\n\t" + "call __do_init\n\t" + ".popsection"); +#elif defined(__arm__) || defined(__aarch64__) +__asm__(".pushsection .init,\"ax\",%progbits\n\t" + "bl __do_init\n\t" + ".popsection"); +#elif defined(__mips__) +__asm__(".pushsection .init,\"ax\",@progbits\n\t" + "jal __do_init\n\t" + ".popsection"); +#elif defined(__powerpc__) || defined(__powerpc64__) +__asm__(".pushsection .init,\"ax\",@progbits\n\t" + "bl __do_init\n\t" + "nop\n\t" + ".popsection"); +#elif defined(__sparc__) +__asm__(".pushsection .init,\"ax\",@progbits\n\t" + "call __do_init\n\t" + ".popsection"); +#else +#error "crtbegin without .init_fini array unimplemented for this architecture" +#endif // CRT_HAS_INITFINI_ARRAY + +#ifndef CRT_HAS_INITFINI_ARRAY +static fp __DTOR_LIST__[] + __attribute__((section(".dtors"), aligned(sizeof(fp)))) = {(fp)-1}; +extern fp __DTOR_LIST_END__[]; +#endif + +static void __attribute__((used)) __do_fini(void) { + static _Bool __finalized; + if (__builtin_expect(__finalized, 0)) + return; + __finalized = 1; + + if (__cxa_finalize) + __cxa_finalize(__dso_handle); + +#ifndef CRT_HAS_INITFINI_ARRAY + const size_t n = __DTOR_LIST_END__ - __DTOR_LIST__ - 1; + for (size_t i = 1; i <= n; i++) __DTOR_LIST__[i](); +#endif +#ifdef EH_USE_FRAME_REGISTRY + if (__deregister_frame_info) + __deregister_frame_info(__EH_FRAME_LIST__); +#endif +} + +#ifdef CRT_HAS_INITFINI_ARRAY +__attribute__((section(".fini_array"), + used)) static void (*__fini)(void) = __do_fini; +#elif defined(__i386__) || defined(__x86_64__) +__asm__(".pushsection .fini,\"ax\",@progbits\n\t" + "call __do_fini\n\t" + ".popsection"); +#elif defined(__arm__) || defined(__aarch64__) +__asm__(".pushsection .fini,\"ax\",%progbits\n\t" + "bl __do_fini\n\t" + ".popsection"); +#elif defined(__mips__) +__asm__(".pushsection .fini,\"ax\",@progbits\n\t" + "jal __do_fini\n\t" + ".popsection"); +#elif defined(__powerpc__) || defined(__powerpc64__) +__asm__(".pushsection .fini,\"ax\",@progbits\n\t" + "bl __do_fini\n\t" + "nop\n\t" + ".popsection"); +#elif defined(__riscv) +__asm__(".pushsection .fini,\"ax\",@progbits\n\t" + "call __do_fini\n\t" + ".popsection"); +#elif defined(__sparc__) +__asm__(".pushsection .fini,\"ax\",@progbits\n\t" + "call __do_fini\n\t" + ".popsection"); +#else +#error "crtbegin without .init_fini array unimplemented for this architecture" +#endif // CRT_HAS_INIT_FINI_ARRAY diff --git a/system/lib/compiler-rt/lib/builtins/crtend.c b/system/lib/compiler-rt/lib/builtins/crtend.c new file mode 100644 index 0000000000000..ebcc60b89a102 --- /dev/null +++ b/system/lib/compiler-rt/lib/builtins/crtend.c @@ -0,0 +1,22 @@ +//===-- crtend.c - End of constructors and destructors --------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include + +// Put 4-byte zero which is the length field in FDE at the end as a terminator. +const int32_t __EH_FRAME_LIST_END__[] + __attribute__((section(".eh_frame"), aligned(sizeof(int32_t)), + visibility("hidden"), used)) = {0}; + +#ifndef CRT_HAS_INITFINI_ARRAY +typedef void (*fp)(void); +fp __CTOR_LIST_END__[] + __attribute__((section(".ctors"), visibility("hidden"), used)) = {0}; +fp __DTOR_LIST_END__[] + __attribute__((section(".dtors"), visibility("hidden"), used)) = {0}; +#endif diff --git a/system/lib/compiler-rt/lib/builtins/divtf3.c b/system/lib/compiler-rt/lib/builtins/divtf3.c index 5bcc9a8e4aa18..bd76763b07d3a 100644 --- a/system/lib/compiler-rt/lib/builtins/divtf3.c +++ b/system/lib/compiler-rt/lib/builtins/divtf3.c @@ -14,7 +14,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) #define NUMBER_OF_HALF_ITERATIONS 4 #define NUMBER_OF_FULL_ITERATIONS 1 diff --git a/system/lib/compiler-rt/lib/builtins/extenddftf2.c b/system/lib/compiler-rt/lib/builtins/extenddftf2.c index ddf470ecd6293..835076be1f208 100644 --- a/system/lib/compiler-rt/lib/builtins/extenddftf2.c +++ b/system/lib/compiler-rt/lib/builtins/extenddftf2.c @@ -9,7 +9,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) #define SRC_DOUBLE #define DST_QUAD #include "fp_extend_impl.inc" diff --git a/system/lib/compiler-rt/lib/builtins/extendhftf2.c b/system/lib/compiler-rt/lib/builtins/extendhftf2.c index aefe9737d34f9..a2cb0f771ee9a 100644 --- a/system/lib/compiler-rt/lib/builtins/extendhftf2.c +++ b/system/lib/compiler-rt/lib/builtins/extendhftf2.c @@ -10,8 +10,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) && \ - defined(COMPILER_RT_HAS_FLOAT16) +#if defined(CRT_HAS_TF_MODE) && defined(COMPILER_RT_HAS_FLOAT16) #define SRC_HALF #define DST_QUAD #include "fp_extend_impl.inc" diff --git a/system/lib/compiler-rt/lib/builtins/extendsftf2.c b/system/lib/compiler-rt/lib/builtins/extendsftf2.c index cf1fd2face20a..0739859bcbc18 100644 --- a/system/lib/compiler-rt/lib/builtins/extendsftf2.c +++ b/system/lib/compiler-rt/lib/builtins/extendsftf2.c @@ -9,7 +9,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) #define SRC_SINGLE #define DST_QUAD #include "fp_extend_impl.inc" diff --git a/system/lib/compiler-rt/lib/builtins/fixtfdi.c b/system/lib/compiler-rt/lib/builtins/fixtfdi.c index fe570e6b37555..d27a99b6f364e 100644 --- a/system/lib/compiler-rt/lib/builtins/fixtfdi.c +++ b/system/lib/compiler-rt/lib/builtins/fixtfdi.c @@ -9,7 +9,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) typedef di_int fixint_t; typedef du_int fixuint_t; #include "fp_fixint_impl.inc" diff --git a/system/lib/compiler-rt/lib/builtins/fixtfsi.c b/system/lib/compiler-rt/lib/builtins/fixtfsi.c index a32bd964caa3b..01e352acc592d 100644 --- a/system/lib/compiler-rt/lib/builtins/fixtfsi.c +++ b/system/lib/compiler-rt/lib/builtins/fixtfsi.c @@ -9,7 +9,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) typedef si_int fixint_t; typedef su_int fixuint_t; #include "fp_fixint_impl.inc" diff --git a/system/lib/compiler-rt/lib/builtins/fixtfti.c b/system/lib/compiler-rt/lib/builtins/fixtfti.c index 19f84ce389074..491fca502113d 100644 --- a/system/lib/compiler-rt/lib/builtins/fixtfti.c +++ b/system/lib/compiler-rt/lib/builtins/fixtfti.c @@ -9,7 +9,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) typedef ti_int fixint_t; typedef tu_int fixuint_t; #include "fp_fixint_impl.inc" diff --git a/system/lib/compiler-rt/lib/builtins/fixunstfdi.c b/system/lib/compiler-rt/lib/builtins/fixunstfdi.c index a0805e63db820..febdb8f5682fa 100644 --- a/system/lib/compiler-rt/lib/builtins/fixunstfdi.c +++ b/system/lib/compiler-rt/lib/builtins/fixunstfdi.c @@ -9,7 +9,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) typedef du_int fixuint_t; #include "fp_fixuint_impl.inc" diff --git a/system/lib/compiler-rt/lib/builtins/fixunstfsi.c b/system/lib/compiler-rt/lib/builtins/fixunstfsi.c index 3a1320ed3e0a2..4efc387df453d 100644 --- a/system/lib/compiler-rt/lib/builtins/fixunstfsi.c +++ b/system/lib/compiler-rt/lib/builtins/fixunstfsi.c @@ -9,7 +9,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) typedef su_int fixuint_t; #include "fp_fixuint_impl.inc" diff --git a/system/lib/compiler-rt/lib/builtins/fixunstfti.c b/system/lib/compiler-rt/lib/builtins/fixunstfti.c index 23cd1ab615a70..fa9e7aa07108b 100644 --- a/system/lib/compiler-rt/lib/builtins/fixunstfti.c +++ b/system/lib/compiler-rt/lib/builtins/fixunstfti.c @@ -9,7 +9,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) typedef tu_int fixuint_t; #include "fp_fixuint_impl.inc" diff --git a/system/lib/compiler-rt/lib/builtins/floatdidf.c b/system/lib/compiler-rt/lib/builtins/floatdidf.c index d37c43b1f2f92..c994aad3f079e 100644 --- a/system/lib/compiler-rt/lib/builtins/floatdidf.c +++ b/system/lib/compiler-rt/lib/builtins/floatdidf.c @@ -50,7 +50,7 @@ COMPILER_RT_ABI double __floatdidf(di_int a) { return 0.0; const unsigned N = sizeof(di_int) * CHAR_BIT; const di_int s = a >> (N - 1); - a = (a ^ s) - s; + a = (du_int)(a ^ s) - s; int sd = N - __builtin_clzll(a); // number of significant digits int e = sd - 1; // exponent if (sd > DBL_MANT_DIG) { diff --git a/system/lib/compiler-rt/lib/builtins/floatdisf.c b/system/lib/compiler-rt/lib/builtins/floatdisf.c index 5c6316431e394..0b62ed8689bc6 100644 --- a/system/lib/compiler-rt/lib/builtins/floatdisf.c +++ b/system/lib/compiler-rt/lib/builtins/floatdisf.c @@ -24,7 +24,7 @@ COMPILER_RT_ABI float __floatdisf(di_int a) { return 0.0F; const unsigned N = sizeof(di_int) * CHAR_BIT; const di_int s = a >> (N - 1); - a = (a ^ s) - s; + a = (du_int)(a ^ s) - s; int sd = N - __builtin_clzll(a); // number of significant digits si_int e = sd - 1; // exponent if (sd > FLT_MANT_DIG) { diff --git a/system/lib/compiler-rt/lib/builtins/floatditf.c b/system/lib/compiler-rt/lib/builtins/floatditf.c index 9b07b65825b88..c6e326a1923a6 100644 --- a/system/lib/compiler-rt/lib/builtins/floatditf.c +++ b/system/lib/compiler-rt/lib/builtins/floatditf.c @@ -15,7 +15,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) COMPILER_RT_ABI fp_t __floatditf(di_int a) { const int aWidth = sizeof a * CHAR_BIT; diff --git a/system/lib/compiler-rt/lib/builtins/floatsitf.c b/system/lib/compiler-rt/lib/builtins/floatsitf.c index 80a4ef08fb0ed..4d5b52f4ed919 100644 --- a/system/lib/compiler-rt/lib/builtins/floatsitf.c +++ b/system/lib/compiler-rt/lib/builtins/floatsitf.c @@ -15,7 +15,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) COMPILER_RT_ABI fp_t __floatsitf(si_int a) { const int aWidth = sizeof a * CHAR_BIT; diff --git a/system/lib/compiler-rt/lib/builtins/floattidf.c b/system/lib/compiler-rt/lib/builtins/floattidf.c index 0a1c04bec82ea..7bfe87f53aa02 100644 --- a/system/lib/compiler-rt/lib/builtins/floattidf.c +++ b/system/lib/compiler-rt/lib/builtins/floattidf.c @@ -29,7 +29,7 @@ COMPILER_RT_ABI double __floattidf(ti_int a) { const ti_int s = a >> (N - 1); a = (a ^ s) - s; int sd = N - __clzti2(a); // number of significant digits - int e = sd - 1; // exponent + si_int e = sd - 1; // exponent if (sd > DBL_MANT_DIG) { // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR diff --git a/system/lib/compiler-rt/lib/builtins/floattisf.c b/system/lib/compiler-rt/lib/builtins/floattisf.c index a8fcdbe14c073..717cb361f075a 100644 --- a/system/lib/compiler-rt/lib/builtins/floattisf.c +++ b/system/lib/compiler-rt/lib/builtins/floattisf.c @@ -28,7 +28,7 @@ COMPILER_RT_ABI float __floattisf(ti_int a) { const ti_int s = a >> (N - 1); a = (a ^ s) - s; int sd = N - __clzti2(a); // number of significant digits - int e = sd - 1; // exponent + si_int e = sd - 1; // exponent if (sd > FLT_MANT_DIG) { // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR diff --git a/system/lib/compiler-rt/lib/builtins/floattitf.c b/system/lib/compiler-rt/lib/builtins/floattitf.c index 196cbdae14e02..fff0755c3bb46 100644 --- a/system/lib/compiler-rt/lib/builtins/floattitf.c +++ b/system/lib/compiler-rt/lib/builtins/floattitf.c @@ -25,7 +25,7 @@ // mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm // mmmm mmmm mmmm -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) COMPILER_RT_ABI fp_t __floattitf(ti_int a) { if (a == 0) return 0.0; @@ -34,7 +34,7 @@ COMPILER_RT_ABI fp_t __floattitf(ti_int a) { a = (a ^ s) - s; int sd = N - __clzti2(a); // number of significant digits int e = sd - 1; // exponent - if (sd > LDBL_MANT_DIG) { + if (sd > TF_MANT_DIG) { // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR // 12345678901234567890123456 @@ -43,27 +43,27 @@ COMPILER_RT_ABI fp_t __floattitf(ti_int a) { // Q = bit LDBL_MANT_DIG bits to the right of 1 // R = "or" of all bits to the right of Q switch (sd) { - case LDBL_MANT_DIG + 1: + case TF_MANT_DIG + 1: a <<= 1; break; - case LDBL_MANT_DIG + 2: + case TF_MANT_DIG + 2: break; default: - a = ((tu_int)a >> (sd - (LDBL_MANT_DIG + 2))) | - ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG + 2) - sd))) != 0); + a = ((tu_int)a >> (sd - (TF_MANT_DIG + 2))) | + ((a & ((tu_int)(-1) >> ((N + TF_MANT_DIG + 2) - sd))) != 0); }; // finish: a |= (a & 4) != 0; // Or P into R ++a; // round - this step may add a significant bit a >>= 2; // dump Q and R // a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits - if (a & ((tu_int)1 << LDBL_MANT_DIG)) { + if (a & ((tu_int)1 << TF_MANT_DIG)) { a >>= 1; ++e; } // a is now rounded to LDBL_MANT_DIG bits } else { - a <<= (LDBL_MANT_DIG - sd); + a <<= (TF_MANT_DIG - sd); // a is now rounded to LDBL_MANT_DIG bits } diff --git a/system/lib/compiler-rt/lib/builtins/floatunditf.c b/system/lib/compiler-rt/lib/builtins/floatunditf.c index 8d310851e179c..abe0ca9ed8c50 100644 --- a/system/lib/compiler-rt/lib/builtins/floatunditf.c +++ b/system/lib/compiler-rt/lib/builtins/floatunditf.c @@ -15,7 +15,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) COMPILER_RT_ABI fp_t __floatunditf(du_int a) { const int aWidth = sizeof a * CHAR_BIT; diff --git a/system/lib/compiler-rt/lib/builtins/floatunsitf.c b/system/lib/compiler-rt/lib/builtins/floatunsitf.c index 7ba1fb6000dcc..3f0a5249fddd3 100644 --- a/system/lib/compiler-rt/lib/builtins/floatunsitf.c +++ b/system/lib/compiler-rt/lib/builtins/floatunsitf.c @@ -15,7 +15,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) COMPILER_RT_ABI fp_t __floatunsitf(su_int a) { const int aWidth = sizeof a * CHAR_BIT; diff --git a/system/lib/compiler-rt/lib/builtins/floatuntidf.c b/system/lib/compiler-rt/lib/builtins/floatuntidf.c index e69e65c1ace4e..4dfca8e493098 100644 --- a/system/lib/compiler-rt/lib/builtins/floatuntidf.c +++ b/system/lib/compiler-rt/lib/builtins/floatuntidf.c @@ -27,7 +27,7 @@ COMPILER_RT_ABI double __floatuntidf(tu_int a) { return 0.0; const unsigned N = sizeof(tu_int) * CHAR_BIT; int sd = N - __clzti2(a); // number of significant digits - int e = sd - 1; // exponent + si_int e = sd - 1; // exponent if (sd > DBL_MANT_DIG) { // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR diff --git a/system/lib/compiler-rt/lib/builtins/floatuntisf.c b/system/lib/compiler-rt/lib/builtins/floatuntisf.c index 9dec0ab5c58fc..a53659cd1fcac 100644 --- a/system/lib/compiler-rt/lib/builtins/floatuntisf.c +++ b/system/lib/compiler-rt/lib/builtins/floatuntisf.c @@ -26,7 +26,7 @@ COMPILER_RT_ABI float __floatuntisf(tu_int a) { return 0.0F; const unsigned N = sizeof(tu_int) * CHAR_BIT; int sd = N - __clzti2(a); // number of significant digits - int e = sd - 1; // exponent + si_int e = sd - 1; // exponent if (sd > FLT_MANT_DIG) { // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR diff --git a/system/lib/compiler-rt/lib/builtins/floatuntitf.c b/system/lib/compiler-rt/lib/builtins/floatuntitf.c index d308d3118d03d..33a81b34eeb19 100644 --- a/system/lib/compiler-rt/lib/builtins/floatuntitf.c +++ b/system/lib/compiler-rt/lib/builtins/floatuntitf.c @@ -25,44 +25,44 @@ // mmmm | mmmm mmmm mmmm mmmm mmmm mmmm mmmm mmmm | mmmm mmmm mmmm mmmm mmmm // mmmm mmmm mmmm -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) COMPILER_RT_ABI fp_t __floatuntitf(tu_int a) { if (a == 0) return 0.0; const unsigned N = sizeof(tu_int) * CHAR_BIT; int sd = N - __clzti2(a); // number of significant digits int e = sd - 1; // exponent - if (sd > LDBL_MANT_DIG) { + if (sd > TF_MANT_DIG) { // start: 0000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQxxxxxxxxxxxxxxxxxx // finish: 000000000000000000000000000000000000001xxxxxxxxxxxxxxxxxxxxxxPQR // 12345678901234567890123456 // 1 = msb 1 bit - // P = bit LDBL_MANT_DIG-1 bits to the right of 1 - // Q = bit LDBL_MANT_DIG bits to the right of 1 + // P = bit TF_MANT_DIG-1 bits to the right of 1 + // Q = bit TF_MANT_DIG bits to the right of 1 // R = "or" of all bits to the right of Q switch (sd) { - case LDBL_MANT_DIG + 1: + case TF_MANT_DIG + 1: a <<= 1; break; - case LDBL_MANT_DIG + 2: + case TF_MANT_DIG + 2: break; default: - a = (a >> (sd - (LDBL_MANT_DIG + 2))) | - ((a & ((tu_int)(-1) >> ((N + LDBL_MANT_DIG + 2) - sd))) != 0); + a = (a >> (sd - (TF_MANT_DIG + 2))) | + ((a & ((tu_int)(-1) >> ((N + TF_MANT_DIG + 2) - sd))) != 0); }; // finish: a |= (a & 4) != 0; // Or P into R ++a; // round - this step may add a significant bit a >>= 2; // dump Q and R - // a is now rounded to LDBL_MANT_DIG or LDBL_MANT_DIG+1 bits - if (a & ((tu_int)1 << LDBL_MANT_DIG)) { + // a is now rounded to TF_MANT_DIG or TF_MANT_DIG+1 bits + if (a & ((tu_int)1 << TF_MANT_DIG)) { a >>= 1; ++e; } - // a is now rounded to LDBL_MANT_DIG bits + // a is now rounded to TF_MANT_DIG bits } else { - a <<= (LDBL_MANT_DIG - sd); - // a is now rounded to LDBL_MANT_DIG bits + a <<= (TF_MANT_DIG - sd); + // a is now rounded to TF_MANT_DIG bits } long_double_bits fb; diff --git a/system/lib/compiler-rt/lib/builtins/fp_lib.h b/system/lib/compiler-rt/lib/builtins/fp_lib.h index 3fb13a033a14c..58eb45fcc7292 100644 --- a/system/lib/compiler-rt/lib/builtins/fp_lib.h +++ b/system/lib/compiler-rt/lib/builtins/fp_lib.h @@ -106,7 +106,13 @@ COMPILER_RT_ABI fp_t __adddf3(fp_t a, fp_t b); #elif defined QUAD_PRECISION #if __LDBL_MANT_DIG__ == 113 && defined(__SIZEOF_INT128__) +// TODO: Availability of the *tf functions should not depend on long double +// being IEEE 128, but instead on being able to use a 128-bit floating-point +// type, which includes __float128. +// Right now this (incorrectly) stops the builtins from being used for x86. #define CRT_LDBL_128BIT +#define CRT_HAS_TF_MODE +#define TF_C(c) c##L typedef uint64_t half_rep_t; typedef __uint128_t rep_t; typedef __int128_t srep_t; @@ -116,6 +122,7 @@ typedef long double fp_t; // Note: Since there is no explicit way to tell compiler the constant is a // 128-bit integer, we let the constant be casted to 128-bit integer #define significandBits 112 +#define TF_MANT_DIG (significandBits + 1) static __inline int rep_clz(rep_t a) { const union { diff --git a/system/lib/compiler-rt/lib/builtins/gcc_personality_v0.c b/system/lib/compiler-rt/lib/builtins/gcc_personality_v0.c index 58fd7ceb58c35..ef63a5fb83472 100644 --- a/system/lib/compiler-rt/lib/builtins/gcc_personality_v0.c +++ b/system/lib/compiler-rt/lib/builtins/gcc_personality_v0.c @@ -219,7 +219,7 @@ COMPILER_RT_ABI _Unwind_Reason_Code __gcc_personality_v0( } // Walk call-site table looking for range that includes current PC. uint8_t callSiteEncoding = *lsda++; - uint32_t callSiteTableLength = readULEB128(&lsda); + size_t callSiteTableLength = readULEB128(&lsda); const uint8_t *callSiteTableStart = lsda; const uint8_t *callSiteTableEnd = callSiteTableStart + callSiteTableLength; const uint8_t *p = callSiteTableStart; diff --git a/system/lib/compiler-rt/lib/builtins/int_mulo_impl.inc b/system/lib/compiler-rt/lib/builtins/int_mulo_impl.inc index 567d8b9e6e603..27e7c8c43d600 100644 --- a/system/lib/compiler-rt/lib/builtins/int_mulo_impl.inc +++ b/system/lib/compiler-rt/lib/builtins/int_mulo_impl.inc @@ -18,10 +18,10 @@ static __inline fixint_t __muloXi4(fixint_t a, fixint_t b, int *overflow) { const int N = (int)(sizeof(fixint_t) * CHAR_BIT); - const fixint_t MIN = (fixint_t)1 << (N - 1); + const fixint_t MIN = (fixint_t)((fixuint_t)1 << (N - 1)); const fixint_t MAX = ~MIN; *overflow = 0; - fixint_t result = a * b; + fixint_t result = (fixuint_t)a * b; if (a == MIN) { if (b != 0 && b != 1) *overflow = 1; diff --git a/system/lib/compiler-rt/lib/builtins/int_mulv_impl.inc b/system/lib/compiler-rt/lib/builtins/int_mulv_impl.inc index 1e920716ec499..06559cf302ea6 100644 --- a/system/lib/compiler-rt/lib/builtins/int_mulv_impl.inc +++ b/system/lib/compiler-rt/lib/builtins/int_mulv_impl.inc @@ -18,7 +18,7 @@ static __inline fixint_t __mulvXi3(fixint_t a, fixint_t b) { const int N = (int)(sizeof(fixint_t) * CHAR_BIT); - const fixint_t MIN = (fixint_t)1 << (N - 1); + const fixint_t MIN = (fixint_t)((fixuint_t)1 << (N - 1)); const fixint_t MAX = ~MIN; if (a == MIN) { if (b == 0 || b == 1) diff --git a/system/lib/compiler-rt/lib/builtins/lshrti3.c b/system/lib/compiler-rt/lib/builtins/lshrti3.c index d00a220959939..5dc8a0a2347f2 100644 --- a/system/lib/compiler-rt/lib/builtins/lshrti3.c +++ b/system/lib/compiler-rt/lib/builtins/lshrti3.c @@ -18,7 +18,7 @@ // Precondition: 0 <= b < bits_in_tword -COMPILER_RT_ABI ti_int __lshrti3(ti_int a, si_int b) { +COMPILER_RT_ABI ti_int __lshrti3(ti_int a, int b) { const int bits_in_dword = (int)(sizeof(di_int) * CHAR_BIT); utwords input; utwords result; diff --git a/system/lib/compiler-rt/lib/builtins/mulodi4.c b/system/lib/compiler-rt/lib/builtins/mulodi4.c index 7209676a327e4..6ecf92664fb5c 100644 --- a/system/lib/compiler-rt/lib/builtins/mulodi4.c +++ b/system/lib/compiler-rt/lib/builtins/mulodi4.c @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #define fixint_t di_int +#define fixuint_t du_int #include "int_mulo_impl.inc" // Returns: a * b diff --git a/system/lib/compiler-rt/lib/builtins/mulosi4.c b/system/lib/compiler-rt/lib/builtins/mulosi4.c index 4e03c24455d67..3fd18a122a463 100644 --- a/system/lib/compiler-rt/lib/builtins/mulosi4.c +++ b/system/lib/compiler-rt/lib/builtins/mulosi4.c @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #define fixint_t si_int +#define fixuint_t su_int #include "int_mulo_impl.inc" // Returns: a * b diff --git a/system/lib/compiler-rt/lib/builtins/muloti4.c b/system/lib/compiler-rt/lib/builtins/muloti4.c index 9a7aa85b022bf..9aab6fc3efb33 100644 --- a/system/lib/compiler-rt/lib/builtins/muloti4.c +++ b/system/lib/compiler-rt/lib/builtins/muloti4.c @@ -19,6 +19,7 @@ // Effects: sets *overflow to 1 if a * b overflows #define fixint_t ti_int +#define fixuint_t tu_int #include "int_mulo_impl.inc" COMPILER_RT_ABI ti_int __muloti4(ti_int a, ti_int b, int *overflow) { diff --git a/system/lib/compiler-rt/lib/builtins/multf3.c b/system/lib/compiler-rt/lib/builtins/multf3.c index 0626fb8c7fc9c..8fd73688712cb 100644 --- a/system/lib/compiler-rt/lib/builtins/multf3.c +++ b/system/lib/compiler-rt/lib/builtins/multf3.c @@ -14,7 +14,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) #include "fp_mul_impl.inc" COMPILER_RT_ABI fp_t __multf3(fp_t a, fp_t b) { return __mulXf3__(a, b); } diff --git a/system/lib/compiler-rt/lib/builtins/mulvdi3.c b/system/lib/compiler-rt/lib/builtins/mulvdi3.c index 1d672c6dc155d..d787d297d564e 100644 --- a/system/lib/compiler-rt/lib/builtins/mulvdi3.c +++ b/system/lib/compiler-rt/lib/builtins/mulvdi3.c @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #define fixint_t di_int +#define fixuint_t du_int #include "int_mulv_impl.inc" // Returns: a * b diff --git a/system/lib/compiler-rt/lib/builtins/mulvsi3.c b/system/lib/compiler-rt/lib/builtins/mulvsi3.c index 00b2e50eeca91..2571881195fcc 100644 --- a/system/lib/compiler-rt/lib/builtins/mulvsi3.c +++ b/system/lib/compiler-rt/lib/builtins/mulvsi3.c @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #define fixint_t si_int +#define fixuint_t su_int #include "int_mulv_impl.inc" // Returns: a * b diff --git a/system/lib/compiler-rt/lib/builtins/mulvti3.c b/system/lib/compiler-rt/lib/builtins/mulvti3.c index ba355149f9a76..fad9b2ae2765b 100644 --- a/system/lib/compiler-rt/lib/builtins/mulvti3.c +++ b/system/lib/compiler-rt/lib/builtins/mulvti3.c @@ -19,6 +19,7 @@ // Effects: aborts if a * b overflows #define fixint_t ti_int +#define fixuint_t tu_int #include "int_mulv_impl.inc" COMPILER_RT_ABI ti_int __mulvti3(ti_int a, ti_int b) { return __mulvXi3(a, b); } diff --git a/system/lib/compiler-rt/lib/builtins/negvdi2.c b/system/lib/compiler-rt/lib/builtins/negvdi2.c index 5c52b3ec2aa60..8c1cf2fa58d46 100644 --- a/system/lib/compiler-rt/lib/builtins/negvdi2.c +++ b/system/lib/compiler-rt/lib/builtins/negvdi2.c @@ -17,7 +17,8 @@ // Effects: aborts if -a overflows COMPILER_RT_ABI di_int __negvdi2(di_int a) { - const di_int MIN = (di_int)1 << ((int)(sizeof(di_int) * CHAR_BIT) - 1); + const di_int MIN = + (di_int)((du_int)1 << ((int)(sizeof(di_int) * CHAR_BIT) - 1)); if (a == MIN) compilerrt_abort(); return -a; diff --git a/system/lib/compiler-rt/lib/builtins/negvsi2.c b/system/lib/compiler-rt/lib/builtins/negvsi2.c index cccdee6dc5e55..70f214f9761d8 100644 --- a/system/lib/compiler-rt/lib/builtins/negvsi2.c +++ b/system/lib/compiler-rt/lib/builtins/negvsi2.c @@ -17,7 +17,8 @@ // Effects: aborts if -a overflows COMPILER_RT_ABI si_int __negvsi2(si_int a) { - const si_int MIN = (si_int)1 << ((int)(sizeof(si_int) * CHAR_BIT) - 1); + const si_int MIN = + (si_int)((su_int)1 << ((int)(sizeof(si_int) * CHAR_BIT) - 1)); if (a == MIN) compilerrt_abort(); return -a; diff --git a/system/lib/compiler-rt/lib/builtins/os_version_check.c b/system/lib/compiler-rt/lib/builtins/os_version_check.c index ebfb2dfc72ddd..182eabe7a6ae2 100644 --- a/system/lib/compiler-rt/lib/builtins/os_version_check.c +++ b/system/lib/compiler-rt/lib/builtins/os_version_check.c @@ -86,6 +86,10 @@ typedef Boolean (*CFStringGetCStringFuncTy)(CFStringRef, char *, CFIndex, CFStringEncoding); typedef void (*CFReleaseFuncTy)(CFTypeRef); +extern __attribute__((weak_import)) +bool _availability_version_check(uint32_t count, + dyld_build_version_t versions[]); + static void _initializeAvailabilityCheck(bool LoadPlist) { if (AvailabilityVersionCheck && !LoadPlist) { // New API is supported and we're not being asked to load the plist, @@ -94,8 +98,8 @@ static void _initializeAvailabilityCheck(bool LoadPlist) { } // Use the new API if it's is available. - AvailabilityVersionCheck = (AvailabilityVersionCheckFuncTy)dlsym( - RTLD_DEFAULT, "_availability_version_check"); + if (_availability_version_check) + AvailabilityVersionCheck = &_availability_version_check; if (AvailabilityVersionCheck && !LoadPlist) { // New API is supported and we're not being asked to load the plist, diff --git a/system/lib/compiler-rt/lib/builtins/powitf2.c b/system/lib/compiler-rt/lib/builtins/powitf2.c index 8e639a03a3c4b..74fe707a4e8c4 100644 --- a/system/lib/compiler-rt/lib/builtins/powitf2.c +++ b/system/lib/compiler-rt/lib/builtins/powitf2.c @@ -13,7 +13,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) // Returns: a ^ b diff --git a/system/lib/compiler-rt/lib/builtins/subtf3.c b/system/lib/compiler-rt/lib/builtins/subtf3.c index 3364c28f81792..e1b1022034bf7 100644 --- a/system/lib/compiler-rt/lib/builtins/subtf3.c +++ b/system/lib/compiler-rt/lib/builtins/subtf3.c @@ -13,7 +13,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) COMPILER_RT_ABI fp_t __addtf3(fp_t a, fp_t b); // Subtraction; flip the sign bit of b and add. diff --git a/system/lib/compiler-rt/lib/builtins/trunctfdf2.c b/system/lib/compiler-rt/lib/builtins/trunctfdf2.c index 6857ea54d8a57..f0d2e4141f3b0 100644 --- a/system/lib/compiler-rt/lib/builtins/trunctfdf2.c +++ b/system/lib/compiler-rt/lib/builtins/trunctfdf2.c @@ -9,7 +9,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) #define SRC_QUAD #define DST_DOUBLE #include "fp_trunc_impl.inc" diff --git a/system/lib/compiler-rt/lib/builtins/trunctfhf2.c b/system/lib/compiler-rt/lib/builtins/trunctfhf2.c index e3a2309d954bc..f7776327251c7 100644 --- a/system/lib/compiler-rt/lib/builtins/trunctfhf2.c +++ b/system/lib/compiler-rt/lib/builtins/trunctfhf2.c @@ -10,8 +10,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) && \ - defined(COMPILER_RT_HAS_FLOAT16) +#if defined(CRT_HAS_TF_MODE) && defined(COMPILER_RT_HAS_FLOAT16) #define SRC_QUAD #define DST_HALF #include "fp_trunc_impl.inc" diff --git a/system/lib/compiler-rt/lib/builtins/trunctfsf2.c b/system/lib/compiler-rt/lib/builtins/trunctfsf2.c index 0261b1e90f5da..242735f738c13 100644 --- a/system/lib/compiler-rt/lib/builtins/trunctfsf2.c +++ b/system/lib/compiler-rt/lib/builtins/trunctfsf2.c @@ -9,7 +9,7 @@ #define QUAD_PRECISION #include "fp_lib.h" -#if defined(CRT_HAS_128BIT) && defined(CRT_LDBL_128BIT) +#if defined(CRT_HAS_TF_MODE) #define SRC_QUAD #define DST_SINGLE #include "fp_trunc_impl.inc" diff --git a/system/lib/compiler-rt/lib/interception/interception.h b/system/lib/compiler-rt/lib/interception/interception.h index 370e4ab844a6b..0bb06c88b9451 100644 --- a/system/lib/compiler-rt/lib/interception/interception.h +++ b/system/lib/compiler-rt/lib/interception/interception.h @@ -14,9 +14,10 @@ #ifndef INTERCEPTION_H #define INTERCEPTION_H +#include "sanitizer_common/sanitizer_asm.h" #include "sanitizer_common/sanitizer_internal_defs.h" -#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_APPLE && \ +#if !SANITIZER_LINUX && !SANITIZER_FREEBSD && !SANITIZER_APPLE && \ !SANITIZER_NETBSD && !SANITIZER_WINDOWS && !SANITIZER_FUCHSIA && \ !SANITIZER_SOLARIS && !SANITIZER_EMSCRIPTEN # error "Interception doesn't work on this operating system." @@ -67,24 +68,50 @@ typedef __sanitizer::OFF64_T OFF64_T; // for more details). To intercept such functions you need to use the // INTERCEPTOR_WITH_SUFFIX(...) macro. -// How it works: -// To replace system functions on Linux we just need to declare functions -// with same names in our library and then obtain the real function pointers +// How it works on Linux +// --------------------- +// +// To replace system functions on Linux we just need to declare functions with +// the same names in our library and then obtain the real function pointers // using dlsym(). -// There is one complication. A user may also intercept some of the functions -// we intercept. To resolve this we declare our interceptors with __interceptor_ -// prefix, and then make actual interceptors weak aliases to __interceptor_ -// functions. // -// This is not so on Mac OS, where the two-level namespace makes -// our replacement functions invisible to other libraries. This may be overcomed -// using the DYLD_FORCE_FLAT_NAMESPACE, but some errors loading the shared -// libraries in Chromium were noticed when doing so. +// There is one complication: a user may also intercept some of the functions we +// intercept. To allow for up to 3 interceptors (including ours) of a given +// function "func", the interceptor implementation is in ___interceptor_func, +// which is aliased by a weak function __interceptor_func, which in turn is +// aliased (via a trampoline) by weak wrapper function "func". +// +// Most user interceptors should define a foreign interceptor as follows: +// +// - provide a non-weak function "func" that performs interception; +// - if __interceptor_func exists, call it to perform the real functionality; +// - if it does not exist, figure out the real function and call it instead. +// +// In rare cases, a foreign interceptor (of another dynamic analysis runtime) +// may be defined as follows (on supported architectures): +// +// - provide a non-weak function __interceptor_func that performs interception; +// - if ___interceptor_func exists, call it to perform the real functionality; +// - if it does not exist, figure out the real function and call it instead; +// - provide a weak function "func" that is an alias to __interceptor_func. +// +// With this protocol, sanitizer interceptors, foreign user interceptors, and +// foreign interceptors of other dynamic analysis runtimes, or any combination +// thereof, may co-exist simultaneously. +// +// How it works on Mac OS +// ---------------------- +// +// This is not so on Mac OS, where the two-level namespace makes our replacement +// functions invisible to other libraries. This may be overcomed using the +// DYLD_FORCE_FLAT_NAMESPACE, but some errors loading the shared libraries in +// Chromium were noticed when doing so. +// // Instead we create a dylib containing a __DATA,__interpose section that // associates library functions with their wrappers. When this dylib is -// preloaded before an executable using DYLD_INSERT_LIBRARIES, it routes all -// the calls to interposed functions done through stubs to the wrapper -// functions. +// preloaded before an executable using DYLD_INSERT_LIBRARIES, it routes all the +// calls to interposed functions done through stubs to the wrapper functions. +// // As it's decided at compile time which functions are to be intercepted on Mac, // INTERCEPT_FUNCTION() is effectively a no-op on this system. @@ -100,58 +127,106 @@ struct interpose_substitution { // For a function foo() create a global pair of pointers { wrap_foo, foo } in // the __DATA,__interpose section. // As a result all the calls to foo() will be routed to wrap_foo() at runtime. -#define INTERPOSER(func_name) __attribute__((used)) \ +#define INTERPOSER(func_name) __attribute__((used)) \ const interpose_substitution substitution_##func_name[] \ __attribute__((section("__DATA, __interpose"))) = { \ - { reinterpret_cast(WRAP(func_name)), \ - reinterpret_cast(func_name) } \ + { reinterpret_cast(WRAP(func_name)), \ + reinterpret_cast(func_name) } \ } // For a function foo() and a wrapper function bar() create a global pair // of pointers { bar, foo } in the __DATA,__interpose section. // As a result all the calls to foo() will be routed to bar() at runtime. #define INTERPOSER_2(func_name, wrapper_name) __attribute__((used)) \ -const interpose_substitution substitution_##func_name[] \ - __attribute__((section("__DATA, __interpose"))) = { \ - { reinterpret_cast(wrapper_name), \ - reinterpret_cast(func_name) } \ +const interpose_substitution substitution_##func_name[] \ + __attribute__((section("__DATA, __interpose"))) = { \ + { reinterpret_cast(wrapper_name), \ + reinterpret_cast(func_name) } \ } # define WRAP(x) wrap_##x -# define WRAPPER_NAME(x) "wrap_"#x +# define TRAMPOLINE(x) WRAP(x) # define INTERCEPTOR_ATTRIBUTE # define DECLARE_WRAPPER(ret_type, func, ...) #elif SANITIZER_WINDOWS # define WRAP(x) __asan_wrap_##x -# define WRAPPER_NAME(x) "__asan_wrap_"#x +# define TRAMPOLINE(x) WRAP(x) # define INTERCEPTOR_ATTRIBUTE __declspec(dllexport) -# define DECLARE_WRAPPER(ret_type, func, ...) \ +# define DECLARE_WRAPPER(ret_type, func, ...) \ extern "C" ret_type func(__VA_ARGS__); -# define DECLARE_WRAPPER_WINAPI(ret_type, func, ...) \ +# define DECLARE_WRAPPER_WINAPI(ret_type, func, ...) \ extern "C" __declspec(dllimport) ret_type __stdcall func(__VA_ARGS__); #elif SANITIZER_EMSCRIPTEN # define WRAP(x) x -# define WRAPPER_NAME(x) #x # define INTERCEPTOR_ATTRIBUTE # define DECLARE_WRAPPER(ret_type, func, ...) -#elif SANITIZER_FREEBSD || SANITIZER_NETBSD -# define WRAP(x) __interceptor_ ## x -# define WRAPPER_NAME(x) "__interceptor_" #x +#elif !SANITIZER_FUCHSIA // LINUX, FREEBSD, NETBSD, SOLARIS # define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default"))) +# if ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT +// Weak aliases of weak aliases do not work, therefore we need to set up a +// trampoline function. The function "func" is a weak alias to the trampoline +// (so that we may check if "func" was overridden), which calls the weak +// function __interceptor_func, which in turn aliases the actual interceptor +// implementation ___interceptor_func: +// +// [wrapper "func": weak] --(alias)--> [TRAMPOLINE(func)] +// | +// +--------(tail call)-------+ +// | +// v +// [__interceptor_func: weak] --(alias)--> [WRAP(func)] +// +// We use inline assembly to define most of this, because not all compilers +// support functions with the "naked" attribute with every architecture. +# define WRAP(x) ___interceptor_ ## x +# define TRAMPOLINE(x) __interceptor_trampoline_ ## x +# if SANITIZER_FREEBSD || SANITIZER_NETBSD // FreeBSD's dynamic linker (incompliantly) gives non-weak symbols higher // priority than weak ones so weak aliases won't work for indirect calls // in position-independent (-fPIC / -fPIE) mode. -# define DECLARE_WRAPPER(ret_type, func, ...) \ - extern "C" ret_type func(__VA_ARGS__) \ - __attribute__((alias("__interceptor_" #func), visibility("default"))); -#elif !SANITIZER_FUCHSIA -# define WRAP(x) __interceptor_ ## x -# define WRAPPER_NAME(x) "__interceptor_" #x -# define INTERCEPTOR_ATTRIBUTE __attribute__((visibility("default"))) -# define DECLARE_WRAPPER(ret_type, func, ...) \ - extern "C" ret_type func(__VA_ARGS__) \ - __attribute__((weak, alias("__interceptor_" #func), visibility("default"))); +# define __ASM_WEAK_WRAPPER(func) ".globl " #func "\n" +# else +# define __ASM_WEAK_WRAPPER(func) ".weak " #func "\n" +# endif // SANITIZER_FREEBSD || SANITIZER_NETBSD +// Keep trampoline implementation in sync with sanitizer_common/sanitizer_asm.h +# define DECLARE_WRAPPER(ret_type, func, ...) \ + extern "C" ret_type func(__VA_ARGS__); \ + extern "C" ret_type TRAMPOLINE(func)(__VA_ARGS__); \ + extern "C" ret_type __interceptor_##func(__VA_ARGS__) \ + INTERCEPTOR_ATTRIBUTE __attribute__((weak)) ALIAS(WRAP(func)); \ + asm( \ + ".text\n" \ + __ASM_WEAK_WRAPPER(func) \ + ".set " #func ", " SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n" \ + ".globl " SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n" \ + ".type " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", %function\n" \ + SANITIZER_STRINGIFY(TRAMPOLINE(func)) ":\n" \ + SANITIZER_STRINGIFY(CFI_STARTPROC) "\n" \ + SANITIZER_STRINGIFY(ASM_TAIL_CALL) " __interceptor_" \ + SANITIZER_STRINGIFY(ASM_PREEMPTIBLE_SYM(func)) "\n" \ + SANITIZER_STRINGIFY(CFI_ENDPROC) "\n" \ + ".size " SANITIZER_STRINGIFY(TRAMPOLINE(func)) ", " \ + ".-" SANITIZER_STRINGIFY(TRAMPOLINE(func)) "\n" \ + ); +# else // ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT +// Some architectures cannot implement efficient interceptor trampolines with +// just a plain jump due to complexities of resolving a preemptible symbol. In +// those cases, revert to just this scheme: +// +// [wrapper "func": weak] --(alias)--> [WRAP(func)] +// +# define WRAP(x) __interceptor_ ## x +# define TRAMPOLINE(x) WRAP(x) +# if SANITIZER_FREEBSD || SANITIZER_NETBSD +# define __ATTRIBUTE_WEAK_WRAPPER +# else +# define __ATTRIBUTE_WEAK_WRAPPER __attribute__((weak)) +# endif // SANITIZER_FREEBSD || SANITIZER_NETBSD +# define DECLARE_WRAPPER(ret_type, func, ...) \ + extern "C" ret_type func(__VA_ARGS__) \ + INTERCEPTOR_ATTRIBUTE __ATTRIBUTE_WEAK_WRAPPER ALIAS(WRAP(func)); +# endif // ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT #endif #if SANITIZER_FUCHSIA @@ -174,10 +249,10 @@ const interpose_substitution substitution_##func_name[] \ # define REAL(x) __interception::PTR_TO_REAL(x) # define FUNC_TYPE(x) x##_type -# define DECLARE_REAL(ret_type, func, ...) \ +# define DECLARE_REAL(ret_type, func, ...) \ typedef ret_type (*FUNC_TYPE(func))(__VA_ARGS__); \ - namespace __interception { \ - extern FUNC_TYPE(func) PTR_TO_REAL(func); \ + namespace __interception { \ + extern FUNC_TYPE(func) PTR_TO_REAL(func); \ } # define ASSIGN_REAL(dst, src) REAL(dst) = REAL(src) #else // SANITIZER_APPLE @@ -188,14 +263,16 @@ const interpose_substitution substitution_##func_name[] \ #endif // SANITIZER_APPLE #if !SANITIZER_FUCHSIA -# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \ +# define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) \ DECLARE_REAL(ret_type, func, __VA_ARGS__) \ + extern "C" ret_type TRAMPOLINE(func)(__VA_ARGS__); \ extern "C" ret_type WRAP(func)(__VA_ARGS__); // Declare an interceptor and its wrapper defined in a different translation // unit (ex. asm). -# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...) \ - extern "C" ret_type WRAP(func)(__VA_ARGS__); \ - extern "C" ret_type func(__VA_ARGS__); +# define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...) \ + extern "C" ret_type TRAMPOLINE(func)(__VA_ARGS__); \ + extern "C" ret_type WRAP(func)(__VA_ARGS__); \ + extern "C" ret_type func(__VA_ARGS__); #else # define DECLARE_REAL_AND_INTERCEPTOR(ret_type, func, ...) # define DECLARE_EXTERN_INTERCEPTOR_AND_WRAPPER(ret_type, func, ...) @@ -227,12 +304,10 @@ const interpose_substitution substitution_##func_name[] \ #elif !SANITIZER_APPLE -#define INTERCEPTOR(ret_type, func, ...) \ - DEFINE_REAL(ret_type, func, __VA_ARGS__) \ - DECLARE_WRAPPER(ret_type, func, __VA_ARGS__) \ - extern "C" \ - INTERCEPTOR_ATTRIBUTE \ - ret_type WRAP(func)(__VA_ARGS__) +#define INTERCEPTOR(ret_type, func, ...) \ + DEFINE_REAL(ret_type, func, __VA_ARGS__) \ + DECLARE_WRAPPER(ret_type, func, __VA_ARGS__) \ + extern "C" INTERCEPTOR_ATTRIBUTE ret_type WRAP(func)(__VA_ARGS__) // We don't need INTERCEPTOR_WITH_SUFFIX on non-Darwin for now. #define INTERCEPTOR_WITH_SUFFIX(ret_type, func, ...) \ @@ -240,10 +315,10 @@ const interpose_substitution substitution_##func_name[] \ #else // SANITIZER_APPLE -#define INTERCEPTOR_ZZZ(suffix, ret_type, func, ...) \ - extern "C" ret_type func(__VA_ARGS__) suffix; \ - extern "C" ret_type WRAP(func)(__VA_ARGS__); \ - INTERPOSER(func); \ +#define INTERCEPTOR_ZZZ(suffix, ret_type, func, ...) \ + extern "C" ret_type func(__VA_ARGS__) suffix; \ + extern "C" ret_type WRAP(func)(__VA_ARGS__); \ + INTERPOSER(func); \ extern "C" INTERCEPTOR_ATTRIBUTE ret_type WRAP(func)(__VA_ARGS__) #define INTERCEPTOR(ret_type, func, ...) \ @@ -258,14 +333,12 @@ const interpose_substitution substitution_##func_name[] \ #endif #if SANITIZER_WINDOWS -# define INTERCEPTOR_WINAPI(ret_type, func, ...) \ +# define INTERCEPTOR_WINAPI(ret_type, func, ...) \ typedef ret_type (__stdcall *FUNC_TYPE(func))(__VA_ARGS__); \ - namespace __interception { \ - FUNC_TYPE(func) PTR_TO_REAL(func); \ - } \ - extern "C" \ - INTERCEPTOR_ATTRIBUTE \ - ret_type __stdcall WRAP(func)(__VA_ARGS__) + namespace __interception { \ + FUNC_TYPE(func) PTR_TO_REAL(func); \ + } \ + extern "C" INTERCEPTOR_ATTRIBUTE ret_type __stdcall WRAP(func)(__VA_ARGS__) #endif // ISO C++ forbids casting between pointer-to-function and pointer-to-object, diff --git a/system/lib/compiler-rt/lib/interception/interception_linux.cpp b/system/lib/compiler-rt/lib/interception/interception_linux.cpp index 5111a87f0a6c9..ef8136eb4fc70 100644 --- a/system/lib/compiler-rt/lib/interception/interception_linux.cpp +++ b/system/lib/compiler-rt/lib/interception/interception_linux.cpp @@ -33,7 +33,7 @@ static int StrCmp(const char *s1, const char *s2) { } #endif -static void *GetFuncAddr(const char *name, uptr wrapper_addr) { +static void *GetFuncAddr(const char *name, uptr trampoline) { #if SANITIZER_NETBSD // FIXME: Find a better way to handle renames if (StrCmp(name, "sigaction")) @@ -50,17 +50,17 @@ static void *GetFuncAddr(const char *name, uptr wrapper_addr) { // In case `name' is not loaded, dlsym ends up finding the actual wrapper. // We don't want to intercept the wrapper and have it point to itself. - if ((uptr)addr == wrapper_addr) + if ((uptr)addr == trampoline) addr = nullptr; } return addr; } bool InterceptFunction(const char *name, uptr *ptr_to_real, uptr func, - uptr wrapper) { - void *addr = GetFuncAddr(name, wrapper); + uptr trampoline) { + void *addr = GetFuncAddr(name, trampoline); *ptr_to_real = (uptr)addr; - return addr && (func == wrapper); + return addr && (func == trampoline); } // dlvsym is a GNU extension supported by some other platforms. @@ -70,12 +70,12 @@ static void *GetFuncAddr(const char *name, const char *ver) { } bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real, - uptr func, uptr wrapper) { + uptr func, uptr trampoline) { void *addr = GetFuncAddr(name, ver); *ptr_to_real = (uptr)addr; - return addr && (func == wrapper); + return addr && (func == trampoline); } -#endif // SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD +# endif // SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD } // namespace __interception diff --git a/system/lib/compiler-rt/lib/interception/interception_linux.h b/system/lib/compiler-rt/lib/interception/interception_linux.h index 6d3957e570267..caa42fc4b7ccd 100644 --- a/system/lib/compiler-rt/lib/interception/interception_linux.h +++ b/system/lib/compiler-rt/lib/interception/interception_linux.h @@ -15,7 +15,7 @@ SANITIZER_SOLARIS || SANITIZER_EMSCRIPTEN #if !defined(INCLUDED_FROM_INTERCEPTION_LIB) -# error "interception_linux.h should be included from interception library only" +# error interception_linux.h should be included from interception library only #endif #ifndef INTERCEPTION_LINUX_H @@ -23,26 +23,26 @@ namespace __interception { bool InterceptFunction(const char *name, uptr *ptr_to_real, uptr func, - uptr wrapper); + uptr trampoline); bool InterceptFunction(const char *name, const char *ver, uptr *ptr_to_real, - uptr func, uptr wrapper); + uptr func, uptr trampoline); } // namespace __interception #define INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) \ ::__interception::InterceptFunction( \ #func, \ - (::__interception::uptr *) & REAL(func), \ - (::__interception::uptr) & (func), \ - (::__interception::uptr) & WRAP(func)) + (::__interception::uptr *)&REAL(func), \ + (::__interception::uptr)&(func), \ + (::__interception::uptr)&TRAMPOLINE(func)) // dlvsym is a GNU extension supported by some other platforms. #if SANITIZER_GLIBC || SANITIZER_FREEBSD || SANITIZER_NETBSD #define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ ::__interception::InterceptFunction( \ #func, symver, \ - (::__interception::uptr *) & REAL(func), \ - (::__interception::uptr) & (func), \ - (::__interception::uptr) & WRAP(func)) + (::__interception::uptr *)&REAL(func), \ + (::__interception::uptr)&(func), \ + (::__interception::uptr)&TRAMPOLINE(func)) #else #define INTERCEPT_FUNCTION_VER_LINUX_OR_FREEBSD(func, symver) \ INTERCEPT_FUNCTION_LINUX_OR_FREEBSD(func) diff --git a/system/lib/compiler-rt/lib/interception/interception_win.cpp b/system/lib/compiler-rt/lib/interception/interception_win.cpp index faaa8ee15381d..00c317510e420 100644 --- a/system/lib/compiler-rt/lib/interception/interception_win.cpp +++ b/system/lib/compiler-rt/lib/interception/interception_win.cpp @@ -141,8 +141,29 @@ static const int kBranchLength = FIRST_32_SECOND_64(kJumpInstructionLength, kIndirectJumpInstructionLength); static const int kDirectBranchLength = kBranchLength + kAddressLength; +# if defined(_MSC_VER) +# define INTERCEPTION_FORMAT(f, a) +# else +# define INTERCEPTION_FORMAT(f, a) __attribute__((format(printf, f, a))) +# endif + +static void (*ErrorReportCallback)(const char *format, ...) + INTERCEPTION_FORMAT(1, 2); + +void SetErrorReportCallback(void (*callback)(const char *format, ...)) { + ErrorReportCallback = callback; +} + +# define ReportError(...) \ + do { \ + if (ErrorReportCallback) \ + ErrorReportCallback(__VA_ARGS__); \ + } while (0) + static void InterceptionFailed() { - // Do we have a good way to abort with an error message here? + ReportError("interception_win: failed due to an unrecoverable error.\n"); + // This acts like an abort when no debugger is attached. According to an old + // comment, calling abort() leads to an infinite recursion in CheckFailed. __debugbreak(); } @@ -249,8 +270,13 @@ static void WritePadding(uptr from, uptr size) { } static void WriteJumpInstruction(uptr from, uptr target) { - if (!DistanceIsWithin2Gig(from + kJumpInstructionLength, target)) + if (!DistanceIsWithin2Gig(from + kJumpInstructionLength, target)) { + ReportError( + "interception_win: cannot write jmp further than 2GB away, from %p to " + "%p.\n", + (void *)from, (void *)target); InterceptionFailed(); + } ptrdiff_t offset = target - from - kJumpInstructionLength; *(u8*)from = 0xE9; *(u32*)(from + 1) = offset; @@ -274,6 +300,10 @@ static void WriteIndirectJumpInstruction(uptr from, uptr indirect_target) { int offset = indirect_target - from - kIndirectJumpInstructionLength; if (!DistanceIsWithin2Gig(from + kIndirectJumpInstructionLength, indirect_target)) { + ReportError( + "interception_win: cannot write indirect jmp with target further than " + "2GB away, from %p to %p.\n", + (void *)from, (void *)indirect_target); InterceptionFailed(); } *(u16*)from = 0x25FF; @@ -492,6 +522,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0xFF8B: // 8B FF : mov edi, edi case 0xEC8B: // 8B EC : mov ebp, esp case 0xc889: // 89 C8 : mov eax, ecx + case 0xE589: // 89 E5 : mov ebp, esp case 0xC18B: // 8B C1 : mov eax, ecx case 0xC033: // 33 C0 : xor eax, eax case 0xC933: // 33 C9 : xor ecx, ecx @@ -641,6 +672,8 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0x24448B: // 8B 44 24 XX : mov eax, dword ptr [esp + XX] case 0x244C8B: // 8B 4C 24 XX : mov ecx, dword ptr [esp + XX] case 0x24548B: // 8B 54 24 XX : mov edx, dword ptr [esp + XX] + case 0x245C8B: // 8B 5C 24 XX : mov ebx, dword ptr [esp + XX] + case 0x246C8B: // 8B 6C 24 XX : mov ebp, dword ptr [esp + XX] case 0x24748B: // 8B 74 24 XX : mov esi, dword ptr [esp + XX] case 0x247C8B: // 8B 7C 24 XX : mov edi, dword ptr [esp + XX] return 4; @@ -652,12 +685,20 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { } #endif - // Unknown instruction! - // FIXME: Unknown instruction failures might happen when we add a new - // interceptor or a new compiler version. In either case, they should result - // in visible and readable error messages. However, merely calling abort() - // leads to an infinite recursion in CheckFailed. - InterceptionFailed(); + // Unknown instruction! This might happen when we add a new interceptor, use + // a new compiler version, or if Windows changed how some functions are + // compiled. In either case, we print the address and 8 bytes of instructions + // to notify the user about the error and to help identify the unknown + // instruction. Don't treat this as a fatal error, though we can break the + // debugger if one has been attached. + u8 *bytes = (u8 *)address; + ReportError( + "interception_win: unhandled instruction at %p: %02x %02x %02x %02x %02x " + "%02x %02x %02x\n", + (void *)address, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], + bytes[5], bytes[6], bytes[7]); + if (::IsDebuggerPresent()) + __debugbreak(); return 0; } @@ -678,6 +719,8 @@ static bool CopyInstructions(uptr to, uptr from, size_t size) { while (cursor != size) { size_t rel_offset = 0; size_t instruction_size = GetInstructionSize(from + cursor, &rel_offset); + if (!instruction_size) + return false; _memcpy((void*)(to + cursor), (void*)(from + cursor), (size_t)instruction_size); if (rel_offset) { @@ -895,6 +938,10 @@ static void **InterestingDLLsAvailable() { "msvcr120.dll", // VS2013 "vcruntime140.dll", // VS2015 "ucrtbase.dll", // Universal CRT +#if (defined(__MINGW32__) && defined(__i386__)) + "libc++.dll", // libc++ + "libunwind.dll", // libunwind +#endif // NTDLL should go last as it exports some functions that we should // override in the CRT [presumably only used internally]. "ntdll.dll", NULL}; diff --git a/system/lib/compiler-rt/lib/interception/interception_win.h b/system/lib/compiler-rt/lib/interception/interception_win.h index 4590013019e37..f6eca82191cba 100644 --- a/system/lib/compiler-rt/lib/interception/interception_win.h +++ b/system/lib/compiler-rt/lib/interception/interception_win.h @@ -41,6 +41,11 @@ bool OverrideImportedFunction(const char *module_to_patch, const char *function_name, uptr new_function, uptr *orig_old_func); +// Sets a callback to be used for reporting errors by interception_win. The +// callback will be called with printf-like arguments. Intended to be used with +// __sanitizer::Report. Pass nullptr to disable error reporting (default). +void SetErrorReportCallback(void (*callback)(const char *format, ...)); + #if !SANITIZER_WINDOWS64 // Exposed for unittests bool OverrideFunctionWithDetour( diff --git a/system/lib/compiler-rt/lib/lsan/lsan.cpp b/system/lib/compiler-rt/lib/lsan/lsan.cpp index 8ec353d071382..6a20a128e021e 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan.cpp +++ b/system/lib/compiler-rt/lib/lsan/lsan.cpp @@ -41,7 +41,7 @@ void __sanitizer::BufferedStackTrace::UnwindImpl( uptr pc, uptr bp, void *context, bool request_fast, u32 max_depth) { using namespace __lsan; uptr stack_top = 0, stack_bottom = 0; - if (ThreadContext *t = CurrentThreadContext()) { + if (ThreadContextLsanBase *t = GetCurrentThread()) { stack_top = t->stack_end(); stack_bottom = t->stack_begin(); } @@ -117,7 +117,7 @@ extern "C" void __lsan_init() { ReplaceSystemMalloc(); InitTlsSize(); InitializeInterceptors(); - InitializeThreadRegistry(); + InitializeThreads(); #if !SANITIZER_EMSCRIPTEN // Emscripten does not have signals InstallDeadlySignalHandlers(LsanOnDeadlySignal); diff --git a/system/lib/compiler-rt/lib/lsan/lsan_allocator.cpp b/system/lib/compiler-rt/lib/lsan/lsan_allocator.cpp index 9ab77a640b7ff..2b5ef0480facf 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_allocator.cpp +++ b/system/lib/compiler-rt/lib/lsan/lsan_allocator.cpp @@ -49,8 +49,11 @@ void InitializeAllocator() { max_malloc_size = kMaxAllowedMallocSize; } +void AllocatorThreadStart() { allocator.InitCache(GetAllocatorCache()); } + void AllocatorThreadFinish() { allocator.SwallowCache(GetAllocatorCache()); + allocator.DestroyCache(GetAllocatorCache()); } static ChunkMetadata *Metadata(const void *p) { @@ -65,12 +68,14 @@ static void RegisterAllocation(const StackTrace &stack, void *p, uptr size) { m->stack_trace_id = StackDepotPut(stack); m->requested_size = size; atomic_store(reinterpret_cast(m), 1, memory_order_relaxed); + RunMallocHooks(p, size); } static void RegisterDeallocation(void *p) { if (!p) return; ChunkMetadata *m = Metadata(p); CHECK(m); + RunFreeHooks(p); atomic_store(reinterpret_cast(m), 0, memory_order_relaxed); } @@ -104,7 +109,6 @@ void *Allocate(const StackTrace &stack, uptr size, uptr alignment, if (cleared && allocator.FromPrimary(p)) memset(p, 0, size); RegisterAllocation(stack, p, size); - RunMallocHooks(p, size); return p; } @@ -119,7 +123,6 @@ static void *Calloc(uptr nmemb, uptr size, const StackTrace &stack) { } void Deallocate(void *p) { - RunFreeHooks(p); RegisterDeallocation(p); allocator.Deallocate(GetAllocatorCache(), p); } @@ -145,6 +148,22 @@ void GetAllocatorCacheRange(uptr *begin, uptr *end) { *end = *begin + sizeof(AllocatorCache); } +static const void *GetMallocBegin(const void *p) { + if (!p) + return nullptr; + void *beg = allocator.GetBlockBegin(p); + if (!beg) + return nullptr; + ChunkMetadata *m = Metadata(beg); + if (!m) + return nullptr; + if (!m->allocated) + return nullptr; + if (m->requested_size == 0) + return nullptr; + return (const void *)beg; +} + uptr GetMallocUsableSize(const void *p) { if (!p) return 0; @@ -153,6 +172,10 @@ uptr GetMallocUsableSize(const void *p) { return m->requested_size; } +uptr GetMallocUsableSizeFast(const void *p) { + return Metadata(p)->requested_size; +} + int lsan_posix_memalign(void **memptr, uptr alignment, uptr size, const StackTrace &stack) { if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) { @@ -275,6 +298,10 @@ uptr GetUserBegin(uptr chunk) { return chunk; } +uptr GetUserAddr(uptr chunk) { + return chunk; +} + LsanMetadata::LsanMetadata(uptr chunk) { metadata_ = Metadata(reinterpret_cast(chunk)); CHECK(metadata_); @@ -304,7 +331,7 @@ void ForEachChunk(ForEachChunkCallback callback, void *arg) { allocator.ForEachChunk(callback, arg); } -IgnoreObjectResult IgnoreObjectLocked(const void *p) { +IgnoreObjectResult IgnoreObject(const void *p) { void *chunk = allocator.GetBlockBegin(p); if (!chunk || p < chunk) return kIgnoreObjectInvalid; ChunkMetadata *m = Metadata(chunk); @@ -319,15 +346,6 @@ IgnoreObjectResult IgnoreObjectLocked(const void *p) { } } -void GetAdditionalThreadContextPtrsLocked(InternalMmapVector *ptrs) { - // This function can be used to treat memory reachable from `tctx` as live. - // This is useful for threads that have been created but not yet started. - - // This is currently a no-op because the LSan `pthread_create()` interceptor - // blocks until the child thread starts which keeps the thread's `arg` pointer - // live. -} - } // namespace __lsan using namespace __lsan; @@ -348,7 +366,7 @@ uptr __sanitizer_get_heap_size() { } SANITIZER_INTERFACE_ATTRIBUTE -uptr __sanitizer_get_free_bytes() { return 0; } +uptr __sanitizer_get_free_bytes() { return 1; } SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_unmapped_bytes() { return 0; } @@ -357,11 +375,29 @@ SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; } SANITIZER_INTERFACE_ATTRIBUTE -int __sanitizer_get_ownership(const void *p) { return Metadata(p) != nullptr; } +int __sanitizer_get_ownership(const void *p) { + return GetMallocBegin(p) != nullptr; +} + +SANITIZER_INTERFACE_ATTRIBUTE +const void * __sanitizer_get_allocated_begin(const void *p) { + return GetMallocBegin(p); +} SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_allocated_size(const void *p) { return GetMallocUsableSize(p); } +SANITIZER_INTERFACE_ATTRIBUTE +uptr __sanitizer_get_allocated_size_fast(const void *p) { + DCHECK_EQ(p, __sanitizer_get_allocated_begin(p)); + uptr ret = GetMallocUsableSizeFast(p); + DCHECK_EQ(ret, __sanitizer_get_allocated_size(p)); + return ret; +} + +SANITIZER_INTERFACE_ATTRIBUTE +void __sanitizer_purge_allocator() { allocator.ForceReleaseToOS(); } + } // extern "C" diff --git a/system/lib/compiler-rt/lib/lsan/lsan_allocator.h b/system/lib/compiler-rt/lib/lsan/lsan_allocator.h index b67d9d7750efc..84cce4c6baebf 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_allocator.h +++ b/system/lib/compiler-rt/lib/lsan/lsan_allocator.h @@ -32,6 +32,7 @@ template void ForEachChunk(const Callable &callback); void GetAllocatorCacheRange(uptr *begin, uptr *end); +void AllocatorThreadStart(); void AllocatorThreadFinish(); void InitializeAllocator(); @@ -68,13 +69,13 @@ using PrimaryAllocator = PrimaryAllocatorASVT; # if SANITIZER_FUCHSIA || defined(__powerpc64__) const uptr kAllocatorSpace = ~(uptr)0; const uptr kAllocatorSize = 0x40000000000ULL; // 4T. -#elif defined(__s390x__) -const uptr kAllocatorSpace = 0x40000000000ULL; -const uptr kAllocatorSize = 0x40000000000ULL; // 4T. -# else +# elif SANITIZER_APPLE const uptr kAllocatorSpace = 0x600000000000ULL; const uptr kAllocatorSize = 0x40000000000ULL; // 4T. -# endif +# else +const uptr kAllocatorSpace = 0x500000000000ULL; +const uptr kAllocatorSize = 0x40000000000ULL; // 4T. +# endif template struct AP64 { // Allocator64 parameters. Deliberately using a short name. static const uptr kSpaceBeg = kAllocatorSpace; diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common.cpp b/system/lib/compiler-rt/lib/lsan/lsan_common.cpp index 8d6f3b9f73a4b..22259e30288fc 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_common.cpp +++ b/system/lib/compiler-rt/lib/lsan/lsan_common.cpp @@ -39,15 +39,13 @@ # else # define OBJC_DATA_MASK 0x00007ffffffffff8UL # endif -// https://github.com/apple-oss-distributions/objc4/blob/8701d5672d3fd3cd817aeb84db1077aafe1a1604/runtime/objc-runtime-new.h#L139 -# define OBJC_FAST_IS_RW 0x8000000000000000UL # endif namespace __lsan { // This mutex is used to prevent races between DoLeakCheck and IgnoreObject, and // also to protect the global list of root regions. -Mutex global_mutex; +static Mutex global_mutex; Flags lsan_flags; @@ -178,13 +176,11 @@ static uptr GetCallerPC(const StackTrace &stack) { } # if SANITIZER_APPLE -// Objective-C class data pointers are stored with flags in the low bits, so -// they need to be transformed back into something that looks like a pointer. -static inline void *MaybeTransformPointer(void *p) { +// Several pointers in the Objective-C runtime (method cache and class_rw_t, +// for example) are tagged with additional bits we need to strip. +static inline void *TransformPointer(void *p) { uptr ptr = reinterpret_cast(p); - if ((ptr & OBJC_FAST_IS_RW) == OBJC_FAST_IS_RW) - ptr &= OBJC_DATA_MASK; - return reinterpret_cast(ptr); + return reinterpret_cast(ptr & OBJC_DATA_MASK); } # endif @@ -254,12 +250,6 @@ static LeakSuppressionContext *GetSuppressionContext() { return suppression_ctx; } -static InternalMmapVectorNoCtor root_regions; - -InternalMmapVectorNoCtor const *GetRootRegions() { - return &root_regions; -} - void InitCommonLsan() { if (common_flags()->detect_leaks) { // Initialization which can fail or print warnings should only be done if @@ -283,13 +273,22 @@ static inline bool MaybeUserPointer(uptr p) { if (p < kMinAddress) return false; # if defined(__x86_64__) - // Accept only canonical form user-space addresses. - return ((p >> 47) == 0); + // TODO: support LAM48 and 5 level page tables. + // LAM_U57 mask format + // * top byte: 0x81 because the format is: [0] [6-bit tag] [0] + // * top-1 byte: 0xff because it should be 0 + // * top-2 byte: 0x80 because Linux uses 128 TB VMA ending at 0x7fffffffffff + constexpr uptr kLAM_U57Mask = 0x81ff80; + constexpr uptr kPointerMask = kLAM_U57Mask << 40; + return ((p & kPointerMask) == 0); # elif defined(__mips64) return ((p >> 40) == 0); # elif defined(__aarch64__) + // TBI (Top Byte Ignore) feature of AArch64: bits [63:56] are ignored in + // address translation and can be used to store a tag. + constexpr uptr kPointerMask = 255ULL << 48; // Accept up to 48 bit VMA. - return ((p >> 48) == 0); + return ((p & kPointerMask) == 0); # elif defined(__loongarch_lp64) // Allow 47-bit user-space VMA at current. return ((p >> 47) == 0); @@ -325,7 +324,7 @@ void ScanRangeForPointers(uptr begin, uptr end, Frontier *frontier, for (; pp + sizeof(void *) <= end; pp += alignment) { // NOLINT void *p = *reinterpret_cast(pp); # if SANITIZER_APPLE - p = MaybeTransformPointer(p); + p = TransformPointer(p); # endif if (!MaybeUserPointer(reinterpret_cast(p))) continue; @@ -557,44 +556,58 @@ static void ProcessThreads(SuspendedThreadsList const &suspended_threads, # endif // SANITIZER_FUCHSIA -void ScanRootRegion(Frontier *frontier, const RootRegion &root_region, - uptr region_begin, uptr region_end, bool is_readable) { - uptr intersection_begin = Max(root_region.begin, region_begin); - uptr intersection_end = Min(region_end, root_region.begin + root_region.size); - if (intersection_begin >= intersection_end) +// A map that contains [region_begin, region_end) pairs. +using RootRegions = DenseMap, uptr>; + +static RootRegions &GetRootRegionsLocked() { + global_mutex.CheckLocked(); + static RootRegions *regions = nullptr; + alignas(RootRegions) static char placeholder[sizeof(RootRegions)]; + if (!regions) + regions = new (placeholder) RootRegions(); + return *regions; +} + +bool HasRootRegions() { return !GetRootRegionsLocked().empty(); } + +void ScanRootRegions(Frontier *frontier, + const InternalMmapVectorNoCtor &mapped_regions) { + if (!flags()->use_root_regions) return; - LOG_POINTERS("Root region %p-%p intersects with mapped region %p-%p (%s)\n", - (void *)root_region.begin, - (void *)(root_region.begin + root_region.size), - (void *)region_begin, (void *)region_end, - is_readable ? "readable" : "unreadable"); - if (is_readable) - ScanRangeForPointers(intersection_begin, intersection_end, frontier, "ROOT", - kReachable); + + InternalMmapVector regions; + GetRootRegionsLocked().forEach([&](const auto &kv) { + regions.push_back({kv.first.first, kv.first.second}); + return true; + }); + + InternalMmapVector intersection; + Intersect(mapped_regions, regions, intersection); + + for (const Region &r : intersection) { + LOG_POINTERS("Root region intersects with mapped region at %p-%p\n", + (void *)r.begin, (void *)r.end); + ScanRangeForPointers(r.begin, r.end, frontier, "ROOT", kReachable); + } } -static void ProcessRootRegion(Frontier *frontier, - const RootRegion &root_region) { +// Scans root regions for heap pointers. +static void ProcessRootRegions(Frontier *frontier) { + if (!flags()->use_root_regions || !HasRootRegions()) + return; #if SANITIZER_EMSCRIPTEN ScanRootRegion(frontier, root_region, 0, emscripten_get_heap_size(), true); #else MemoryMappingLayout proc_maps(/*cache_enabled*/ true); MemoryMappedSegment segment; - while (proc_maps.Next(&segment)) { - ScanRootRegion(frontier, root_region, segment.start, segment.end, - segment.IsReadable()); - } + InternalMmapVector mapped_regions; + while (proc_maps.Next(&segment)) + if (segment.IsReadable()) + mapped_regions.push_back({segment.start, segment.end}); + ScanRootRegions(frontier, mapped_regions); #endif // SANITIZER_EMSCRIPTEN } -// Scans root regions for heap pointers. -static void ProcessRootRegions(Frontier *frontier) { - if (!flags()->use_root_regions) - return; - for (uptr i = 0; i < root_regions.size(); i++) - ProcessRootRegion(frontier, root_regions[i]); -} - static void FloodFillTag(Frontier *frontier, ChunkTag tag) { while (frontier->size()) { uptr next_chunk = frontier->back(); @@ -897,7 +910,7 @@ void LeakReport::AddLeakedChunks(const LeakedChunks &chunks) { leaks_.push_back(leak); } if (flags()->report_objects) { - LeakedObject obj = {leaks_[i].id, chunk, leaked_size}; + LeakedObject obj = {leaks_[i].id, GetUserAddr(chunk), leaked_size}; leaked_objects_.push_back(obj); } } @@ -982,7 +995,7 @@ void LeakReport::PrintSummary() { uptr LeakReport::ApplySuppressions() { LeakSuppressionContext *suppressions = GetSuppressionContext(); - uptr new_suppressions = false; + uptr new_suppressions = 0; for (uptr i = 0; i < leaks_.size(); i++) { if (suppressions->Suppress(leaks_[i].stack_trace_id, leaks_[i].hit_count, leaks_[i].total_size)) { @@ -1031,7 +1044,7 @@ void __lsan_ignore_object(const void *p) { // Cannot use PointsIntoChunk or LsanMetadata here, since the allocator is not // locked. Lock l(&global_mutex); - IgnoreObjectResult res = IgnoreObjectLocked(p); + IgnoreObjectResult res = IgnoreObject(p); if (res == kIgnoreObjectInvalid) VReport(1, "__lsan_ignore_object(): no heap object found at %p\n", p); if (res == kIgnoreObjectAlreadyIgnored) @@ -1047,36 +1060,37 @@ void __lsan_ignore_object(const void *p) { SANITIZER_INTERFACE_ATTRIBUTE void __lsan_register_root_region(const void *begin, uptr size) { #if CAN_SANITIZE_LEAKS - Lock l(&global_mutex); - RootRegion region = {reinterpret_cast(begin), size}; - root_regions.push_back(region); VReport(1, "Registered root region at %p of size %zu\n", begin, size); + uptr b = reinterpret_cast(begin); + uptr e = b + size; + CHECK_LT(b, e); + + Lock l(&global_mutex); + ++GetRootRegionsLocked()[{b, e}]; #endif // CAN_SANITIZE_LEAKS } SANITIZER_INTERFACE_ATTRIBUTE void __lsan_unregister_root_region(const void *begin, uptr size) { #if CAN_SANITIZE_LEAKS - Lock l(&global_mutex); - bool removed = false; - for (uptr i = 0; i < root_regions.size(); i++) { - RootRegion region = root_regions[i]; - if (region.begin == reinterpret_cast(begin) && region.size == size) { - removed = true; - uptr last_index = root_regions.size() - 1; - root_regions[i] = root_regions[last_index]; - root_regions.pop_back(); - VReport(1, "Unregistered root region at %p of size %zu\n", begin, size); - break; + uptr b = reinterpret_cast(begin); + uptr e = b + size; + CHECK_LT(b, e); + VReport(1, "Unregistered root region at %p of size %zu\n", begin, size); + + { + Lock l(&global_mutex); + if (auto *f = GetRootRegionsLocked().find({b, e})) { + if (--(f->second) == 0) + GetRootRegionsLocked().erase(f); + return; } } - if (!removed) { - Report( - "__lsan_unregister_root_region(): region at %p of size %zu has not " - "been registered.\n", - begin, size); - Die(); - } + Report( + "__lsan_unregister_root_region(): region at %p of size %zu has not " + "been registered.\n", + begin, size); + Die(); #endif // CAN_SANITIZE_LEAKS } diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common.h b/system/lib/compiler-rt/lib/lsan/lsan_common.h index 86eb7d84a8891..46b7e6f34025c 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_common.h +++ b/system/lib/compiler-rt/lib/lsan/lsan_common.h @@ -18,6 +18,7 @@ #include "sanitizer_common/sanitizer_common.h" #include "sanitizer_common/sanitizer_internal_defs.h" #include "sanitizer_common/sanitizer_platform.h" +#include "sanitizer_common/sanitizer_range.h" #include "sanitizer_common/sanitizer_stackdepot.h" #include "sanitizer_common/sanitizer_stoptheworld.h" #include "sanitizer_common/sanitizer_symbolizer.h" @@ -79,11 +80,6 @@ enum IgnoreObjectResult { kIgnoreObjectInvalid }; -struct Range { - uptr begin; - uptr end; -}; - //// -------------------------------------------------------------------------- //// Poisoning prototypes. //// -------------------------------------------------------------------------- @@ -96,8 +92,8 @@ bool WordIsPoisoned(uptr addr); //// -------------------------------------------------------------------------- // Wrappers for ThreadRegistry access. -void LockThreadRegistry() SANITIZER_NO_THREAD_SAFETY_ANALYSIS; -void UnlockThreadRegistry() SANITIZER_NO_THREAD_SAFETY_ANALYSIS; +void LockThreads() SANITIZER_NO_THREAD_SAFETY_ANALYSIS; +void UnlockThreads() SANITIZER_NO_THREAD_SAFETY_ANALYSIS; // If called from the main thread, updates the main thread's TID in the thread // registry. We need this to handle processes that fork() without a subsequent // exec(), which invalidates the recorded TID. To update it, we must call @@ -131,6 +127,9 @@ void GetAllocatorGlobalRange(uptr *begin, uptr *end); uptr PointsIntoChunk(void *p); // Returns address of user-visible chunk contained in this allocator chunk. uptr GetUserBegin(uptr chunk); +// Returns user-visible address for chunk. If memory tagging is used this +// function will return the tagged address. +uptr GetUserAddr(uptr chunk); // Wrapper for chunk metadata operations. class LsanMetadata { @@ -151,19 +150,19 @@ class LsanMetadata { void ForEachChunk(ForEachChunkCallback callback, void *arg); // Helper for __lsan_ignore_object(). -IgnoreObjectResult IgnoreObjectLocked(const void *p); +IgnoreObjectResult IgnoreObject(const void *p); // The rest of the LSan interface which is implemented by library. struct ScopedStopTheWorldLock { ScopedStopTheWorldLock() { - LockThreadRegistry(); + LockThreads(); LockAllocator(); } ~ScopedStopTheWorldLock() { UnlockAllocator(); - UnlockThreadRegistry(); + UnlockThreads(); } ScopedStopTheWorldLock &operator=(const ScopedStopTheWorldLock &) = delete; @@ -236,11 +235,6 @@ void InitializePlatformSpecificModules(); void ProcessGlobalRegions(Frontier *frontier); void ProcessPlatformSpecificAllocations(Frontier *frontier); -struct RootRegion { - uptr begin; - uptr size; -}; - // LockStuffAndStopTheWorld can start to use Scan* calls to collect into // this Frontier vector before the StopTheWorldCallback actually runs. // This is used when the OS has a unified callback API for suspending @@ -253,9 +247,11 @@ struct CheckForLeaksParam { bool success = false; }; -InternalMmapVectorNoCtor const *GetRootRegions(); -void ScanRootRegion(Frontier *frontier, RootRegion const ®ion, - uptr region_begin, uptr region_end, bool is_readable); +using Region = Range; + +bool HasRootRegions(); +void ScanRootRegions(Frontier *frontier, + const InternalMmapVectorNoCtor ®ion); // Run stoptheworld while holding any platform-specific locks, as well as the // allocator and thread registry locks. void LockStuffAndStopTheWorld(StopTheWorldCallback callback, diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common_mac.cpp b/system/lib/compiler-rt/lib/lsan/lsan_common_mac.cpp index b6b15095744d7..4e5198979b954 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_common_mac.cpp +++ b/system/lib/compiler-rt/lib/lsan/lsan_common_mac.cpp @@ -25,6 +25,8 @@ # include "sanitizer_common/sanitizer_allocator_internal.h" namespace __lsan { +class ThreadContextLsanBase; + enum class SeenRegion { None = 0, AllocOnce = 1 << 0, @@ -50,18 +52,18 @@ struct RegionScanState { typedef struct { int disable_counter; - u32 current_thread_id; + ThreadContextLsanBase *current_thread; AllocatorCache cache; } thread_local_data_t; static pthread_key_t key; static pthread_once_t key_once = PTHREAD_ONCE_INIT; -// The main thread destructor requires the current thread id, -// so we can't destroy it until it's been used and reset to invalid tid +// The main thread destructor requires the current thread, +// so we can't destroy it until it's been used and reset. void restore_tid_data(void *ptr) { thread_local_data_t *data = (thread_local_data_t *)ptr; - if (data->current_thread_id != kInvalidTid) + if (data->current_thread) pthread_setspecific(key, data); } @@ -76,7 +78,7 @@ static thread_local_data_t *get_tls_val(bool alloc) { if (ptr == NULL && alloc) { ptr = (thread_local_data_t *)InternalAlloc(sizeof(*ptr)); ptr->disable_counter = 0; - ptr->current_thread_id = kInvalidTid; + ptr->current_thread = nullptr; ptr->cache = AllocatorCache(); pthread_setspecific(key, ptr); } @@ -99,12 +101,14 @@ void EnableInThisThread() { --*disable_counter; } -u32 GetCurrentThread() { +ThreadContextLsanBase *GetCurrentThread() { thread_local_data_t *data = get_tls_val(false); - return data ? data->current_thread_id : kInvalidTid; + return data ? data->current_thread : nullptr; } -void SetCurrentThread(u32 tid) { get_tls_val(true)->current_thread_id = tid; } +void SetCurrentThread(ThreadContextLsanBase *tctx) { + get_tls_val(true)->current_thread = tctx; +} AllocatorCache *GetAllocatorCache() { return &get_tls_val(true)->cache; } @@ -161,7 +165,8 @@ void ProcessPlatformSpecificAllocations(Frontier *frontier) { vm_address_t address = 0; kern_return_t err = KERN_SUCCESS; - InternalMmapVectorNoCtor const *root_regions = GetRootRegions(); + InternalMmapVector mapped_regions; + bool use_root_regions = flags()->use_root_regions && HasRootRegions(); RegionScanState scan_state; while (err == KERN_SUCCESS) { @@ -199,8 +204,7 @@ void ProcessPlatformSpecificAllocations(Frontier *frontier) { // Recursing over the full memory map is very slow, break out // early if we don't need the full iteration. - if (scan_state.seen_regions == SeenRegion::All && - !(flags()->use_root_regions && root_regions->size() > 0)) { + if (scan_state.seen_regions == SeenRegion::All && !use_root_regions) { break; } @@ -211,15 +215,12 @@ void ProcessPlatformSpecificAllocations(Frontier *frontier) { // // TODO(fjricci) - remove this once sanitizer_procmaps_mac has the same // behavior as sanitizer_procmaps_linux and traverses all memory regions - if (flags()->use_root_regions) { - for (uptr i = 0; i < root_regions->size(); i++) { - ScanRootRegion(frontier, (*root_regions)[i], address, end_address, - info.protection & kProtectionRead); - } - } + if (use_root_regions && (info.protection & kProtectionRead)) + mapped_regions.push_back({address, end_address}); address = end_address; } + ScanRootRegions(frontier, mapped_regions); } // On darwin, we can intercept _exit gracefully, and return a failing exit code diff --git a/system/lib/compiler-rt/lib/lsan/lsan_fuchsia.cpp b/system/lib/compiler-rt/lib/lsan/lsan_fuchsia.cpp index 03ac0afbabff7..4edac9757a9c4 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_fuchsia.cpp +++ b/system/lib/compiler-rt/lib/lsan/lsan_fuchsia.cpp @@ -46,6 +46,7 @@ struct OnStartedArgs { }; void ThreadContext::OnStarted(void *arg) { + ThreadContextLsanBase::OnStarted(arg); auto args = reinterpret_cast(arg); cache_begin_ = args->cache_begin; cache_end_ = args->cache_end; @@ -98,7 +99,7 @@ void *__sanitizer_before_thread_create_hook(thrd_t thread, bool detached, OnCreatedArgs args; args.stack_begin = reinterpret_cast(stack_base); args.stack_end = args.stack_begin + stack_size; - u32 parent_tid = GetCurrentThread(); + u32 parent_tid = GetCurrentThreadId(); u32 tid = ThreadCreate(parent_tid, detached, &args); return reinterpret_cast(static_cast(tid)); } diff --git a/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp b/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp index 996cf50f8f094..10494cee230eb 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp +++ b/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp @@ -208,7 +208,7 @@ INTERCEPTOR(void*, pvalloc, uptr size) { #endif // SANITIZER_INTERCEPT_PVALLOC #if SANITIZER_INTERCEPT_CFREE -INTERCEPTOR(void, cfree, void *p) ALIAS(WRAPPER_NAME(free)); +INTERCEPTOR(void, cfree, void *p) ALIAS(WRAP(free)); #define LSAN_MAYBE_INTERCEPT_CFREE INTERCEPT_FUNCTION(cfree) #else #define LSAN_MAYBE_INTERCEPT_CFREE @@ -426,16 +426,10 @@ INTERCEPTOR(char *, strerror, int errnum) { #if SANITIZER_POSIX -struct ThreadParam { - void *(*callback)(void *arg); - void *param; - atomic_uintptr_t tid; -}; - -extern "C" void *__lsan_thread_start_func(void *arg) { - ThreadParam *p = (ThreadParam*)arg; - void* (*callback)(void *arg) = p->callback; - void *param = p->param; +template +static void *ThreadStartFunc(void *arg) { + u32 parent_tid = (uptr)arg; + uptr tid = ThreadCreate(parent_tid, Detached); // Wait until the last iteration to maximize the chance that we are the last // destructor to run. #if !SANITIZER_NETBSD && !SANITIZER_FREEBSD @@ -444,74 +438,103 @@ extern "C" void *__lsan_thread_start_func(void *arg) { Report("LeakSanitizer: failed to set thread key.\n"); Die(); } -#endif - int tid = 0; - while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0) - internal_sched_yield(); +# endif ThreadStart(tid, GetTid()); -#if SANITIZER_EMSCRIPTEN - emscripten_builtin_free(p); -#else - atomic_store(&p->tid, 0, memory_order_release); -#endif - return callback(param); + auto self = GetThreadSelf(); + auto args = GetThreadArgRetval().GetArgs(self); + void *retval = (*args.routine)(args.arg_retval); + GetThreadArgRetval().Finish(self, retval); + return retval; } INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void *), void *param) { ENSURE_LSAN_INITED; EnsureMainThreadIDIsCorrect(); + + bool detached = [attr]() { + int d = 0; + return attr && !pthread_attr_getdetachstate(attr, &d) && IsStateDetached(d); + }(); + __sanitizer_pthread_attr_t myattr; if (!attr || attr == __ATTRP_C11_THREAD) { pthread_attr_init(&myattr); attr = &myattr; } AdjustStackSize(attr); - int detached = 0; - pthread_attr_getdetachstate(attr, &detached); -#if SANITIZER_EMSCRIPTEN - ThreadParam *p = (ThreadParam *) emscripten_builtin_malloc(sizeof(ThreadParam)); - p->callback = callback; - p->param = param; - atomic_store(&p->tid, 0, memory_order_relaxed); -#else - ThreadParam p; - p.callback = callback; - p.param = param; - atomic_store(&p.tid, 0, memory_order_relaxed); -#endif - int res; + uptr this_tid = GetCurrentThreadId(); + int result; { // Ignore all allocations made by pthread_create: thread stack/TLS may be // stored by pthread for future reuse even after thread destruction, and // the linked list it's stored in doesn't even hold valid pointers to the // objects, the latter are calculated by obscure pointer arithmetic. ScopedInterceptorDisabler disabler; -#if SANITIZER_EMSCRIPTEN - res = REAL(pthread_create)(th, attr, __lsan_thread_start_func, p); -#else - res = REAL(pthread_create)(th, attr, __lsan_thread_start_func, &p); -#endif - } - if (res == 0) { - int tid = ThreadCreate(GetCurrentThread(), IsStateDetached(detached)); - CHECK_NE(tid, kMainTid); -#if SANITIZER_EMSCRIPTEN - atomic_store(&p->tid, tid, memory_order_release); -#else - atomic_store(&p.tid, tid, memory_order_release); - while (atomic_load(&p.tid, memory_order_acquire) != 0) - internal_sched_yield(); -#endif + GetThreadArgRetval().Create(detached, {callback, param}, [&]() -> uptr { + result = REAL(pthread_create)( + th, attr, detached ? ThreadStartFunc : ThreadStartFunc, + (void *)this_tid); + return result ? 0 : *(uptr *)(th); + }); } if (attr == &myattr) pthread_attr_destroy(&myattr); - return res; -} - -INTERCEPTOR(int, pthread_join, void *t, void **arg) { - return REAL(pthread_join)(t, arg); -} + return result; +} + +INTERCEPTOR(int, pthread_join, void *thread, void **retval) { + int result; + GetThreadArgRetval().Join((uptr)thread, [&]() { + result = REAL(pthread_join)(thread, retval); + return !result; + }); + return result; +} + +INTERCEPTOR(int, pthread_detach, void *thread) { + int result; + GetThreadArgRetval().Detach((uptr)thread, [&]() { + result = REAL(pthread_detach)(thread); + return !result; + }); + return result; +} + +INTERCEPTOR(int, pthread_exit, void *retval) { + GetThreadArgRetval().Finish(GetThreadSelf(), retval); + return REAL(pthread_exit)(retval); +} + +# if SANITIZER_INTERCEPT_TRYJOIN +INTERCEPTOR(int, pthread_tryjoin_np, void *thread, void **ret) { + int result; + GetThreadArgRetval().Join((uptr)thread, [&]() { + result = REAL(pthread_tryjoin_np)(thread, ret); + return !result; + }); + return result; +} +# define LSAN_MAYBE_INTERCEPT_TRYJOIN INTERCEPT_FUNCTION(pthread_tryjoin_np) +# else +# define LSAN_MAYBE_INTERCEPT_TRYJOIN +# endif // SANITIZER_INTERCEPT_TRYJOIN + +# if SANITIZER_INTERCEPT_TIMEDJOIN +INTERCEPTOR(int, pthread_timedjoin_np, void *thread, void **ret, + const struct timespec *abstime) { + int result; + GetThreadArgRetval().Join((uptr)thread, [&]() { + result = REAL(pthread_timedjoin_np)(thread, ret, abstime); + return !result; + }); + return result; +} +# define LSAN_MAYBE_INTERCEPT_TIMEDJOIN \ + INTERCEPT_FUNCTION(pthread_timedjoin_np) +# else +# define LSAN_MAYBE_INTERCEPT_TIMEDJOIN +# endif // SANITIZER_INTERCEPT_TIMEDJOIN DEFINE_REAL_PTHREAD_FUNCTIONS @@ -522,6 +545,7 @@ INTERCEPTOR(void, _exit, int status) { } #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) +#define SIGNAL_INTERCEPTOR_ENTER() ENSURE_LSAN_INITED #include "sanitizer_common/sanitizer_signal_interceptors.inc" #endif @@ -549,6 +573,10 @@ void InitializeInterceptors() { LSAN_MAYBE_INTERCEPT_MALLOPT; INTERCEPT_FUNCTION(pthread_create); INTERCEPT_FUNCTION(pthread_join); + INTERCEPT_FUNCTION(pthread_detach); + INTERCEPT_FUNCTION(pthread_exit); + LSAN_MAYBE_INTERCEPT_TIMEDJOIN; + LSAN_MAYBE_INTERCEPT_TRYJOIN; INTERCEPT_FUNCTION(_exit); LSAN_MAYBE_INTERCEPT__LWP_EXIT; diff --git a/system/lib/compiler-rt/lib/lsan/lsan_linux.cpp b/system/lib/compiler-rt/lib/lsan/lsan_linux.cpp index 63ae2c1a945b8..26aaf190ce873 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_linux.cpp +++ b/system/lib/compiler-rt/lib/lsan/lsan_linux.cpp @@ -14,13 +14,14 @@ #if SANITIZER_LINUX || SANITIZER_NETBSD || SANITIZER_FUCHSIA || SANITIZER_EMSCRIPTEN -#include "lsan_allocator.h" +# include "lsan_allocator.h" +# include "lsan_thread.h" namespace __lsan { -static THREADLOCAL u32 current_thread_tid = kInvalidTid; -u32 GetCurrentThread() { return current_thread_tid; } -void SetCurrentThread(u32 tid) { current_thread_tid = tid; } +static THREADLOCAL ThreadContextLsanBase *current_thread = nullptr; +ThreadContextLsanBase *GetCurrentThread() { return current_thread; } +void SetCurrentThread(ThreadContextLsanBase *tctx) { current_thread = tctx; } static THREADLOCAL AllocatorCache allocator_cache; AllocatorCache *GetAllocatorCache() { return &allocator_cache; } diff --git a/system/lib/compiler-rt/lib/lsan/lsan_mac.cpp b/system/lib/compiler-rt/lib/lsan/lsan_mac.cpp index 6964a9ba28df5..990954a8b6879 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_mac.cpp +++ b/system/lib/compiler-rt/lib/lsan/lsan_mac.cpp @@ -67,10 +67,9 @@ typedef struct { ALWAYS_INLINE void lsan_register_worker_thread(int parent_tid) { - if (GetCurrentThread() == kInvalidTid) { + if (GetCurrentThreadId() == kInvalidTid) { u32 tid = ThreadCreate(parent_tid, true); ThreadStart(tid, GetTid()); - SetCurrentThread(tid); } } @@ -81,7 +80,7 @@ extern "C" void lsan_dispatch_call_block_and_release(void *block) { VReport(2, "lsan_dispatch_call_block_and_release(): " "context: %p, pthread_self: %p\n", - block, pthread_self()); + block, (void*)pthread_self()); lsan_register_worker_thread(context->parent_tid); // Call the original dispatcher for the block. context->func(context->block); @@ -101,7 +100,7 @@ extern "C" lsan_block_context_t *alloc_lsan_context(void *ctxt, (lsan_block_context_t *)lsan_malloc(sizeof(lsan_block_context_t), stack); lsan_ctxt->block = ctxt; lsan_ctxt->func = func; - lsan_ctxt->parent_tid = GetCurrentThread(); + lsan_ctxt->parent_tid = GetCurrentThreadId(); return lsan_ctxt; } @@ -146,13 +145,13 @@ void dispatch_source_set_event_handler(dispatch_source_t ds, void (^work)(void)); } -#define GET_LSAN_BLOCK(work) \ - void (^lsan_block)(void); \ - int parent_tid = GetCurrentThread(); \ - lsan_block = ^(void) { \ - lsan_register_worker_thread(parent_tid); \ - work(); \ - } +# define GET_LSAN_BLOCK(work) \ + void (^lsan_block)(void); \ + int parent_tid = GetCurrentThreadId(); \ + lsan_block = ^(void) { \ + lsan_register_worker_thread(parent_tid); \ + work(); \ + } INTERCEPTOR(void, dispatch_async, dispatch_queue_t dq, void (^work)(void)) { GET_LSAN_BLOCK(work); diff --git a/system/lib/compiler-rt/lib/lsan/lsan_posix.cpp b/system/lib/compiler-rt/lib/lsan/lsan_posix.cpp index 3c7bc15a851af..d99e1cc0105ef 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_posix.cpp +++ b/system/lib/compiler-rt/lib/lsan/lsan_posix.cpp @@ -35,6 +35,7 @@ struct OnStartedArgs { }; void ThreadContext::OnStarted(void *arg) { + ThreadContextLsanBase::OnStarted(arg); auto args = reinterpret_cast(arg); stack_begin_ = args->stack_begin; stack_end_ = args->stack_end; @@ -88,7 +89,7 @@ static void OnStackUnwind(const SignalContext &sig, const void *, } void LsanOnDeadlySignal(int signo, void *siginfo, void *context) { - HandleDeadlySignal(siginfo, context, GetCurrentThread(), &OnStackUnwind, + HandleDeadlySignal(siginfo, context, GetCurrentThreadId(), &OnStackUnwind, nullptr); } diff --git a/system/lib/compiler-rt/lib/lsan/lsan_thread.cpp b/system/lib/compiler-rt/lib/lsan/lsan_thread.cpp index 2b694ba5e4071..f27e36f2e9ca4 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_thread.cpp +++ b/system/lib/compiler-rt/lib/lsan/lsan_thread.cpp @@ -24,24 +24,41 @@ namespace __lsan { static ThreadRegistry *thread_registry; +static ThreadArgRetval *thread_arg_retval; + +static Mutex mu_for_thread_context; +static LowLevelAllocator allocator_for_thread_context; static ThreadContextBase *CreateThreadContext(u32 tid) { - void *mem = MmapOrDie(sizeof(ThreadContext), "ThreadContext"); - return new (mem) ThreadContext(tid); + Lock lock(&mu_for_thread_context); + return new (allocator_for_thread_context) ThreadContext(tid); } -void InitializeThreadRegistry() { - static ALIGNED(64) char thread_registry_placeholder[sizeof(ThreadRegistry)]; +void InitializeThreads() { + static ALIGNED(alignof( + ThreadRegistry)) char thread_registry_placeholder[sizeof(ThreadRegistry)]; thread_registry = new (thread_registry_placeholder) ThreadRegistry(CreateThreadContext); + + static ALIGNED(alignof(ThreadArgRetval)) char + thread_arg_retval_placeholder[sizeof(ThreadArgRetval)]; + thread_arg_retval = new (thread_arg_retval_placeholder) ThreadArgRetval(); } +ThreadArgRetval &GetThreadArgRetval() { return *thread_arg_retval; } + ThreadContextLsanBase::ThreadContextLsanBase(int tid) : ThreadContextBase(tid) {} +void ThreadContextLsanBase::OnStarted(void *arg) { + SetCurrentThread(this); + AllocatorThreadStart(); +} + void ThreadContextLsanBase::OnFinished() { AllocatorThreadFinish(); DTLS_Destroy(); + SetCurrentThread(nullptr); } u32 ThreadCreate(u32 parent_tid, bool detached, void *arg) { @@ -51,26 +68,13 @@ u32 ThreadCreate(u32 parent_tid, bool detached, void *arg) { void ThreadContextLsanBase::ThreadStart(u32 tid, tid_t os_id, ThreadType thread_type, void *arg) { thread_registry->StartThread(tid, os_id, thread_type, arg); - SetCurrentThread(tid); -} - -void ThreadFinish() { - thread_registry->FinishThread(GetCurrentThread()); - SetCurrentThread(kInvalidTid); } -ThreadContext *CurrentThreadContext() { - if (!thread_registry) - return nullptr; - if (GetCurrentThread() == kInvalidTid) - return nullptr; - // No lock needed when getting current thread. - return (ThreadContext *)thread_registry->GetThreadLocked(GetCurrentThread()); -} +void ThreadFinish() { thread_registry->FinishThread(GetCurrentThreadId()); } void EnsureMainThreadIDIsCorrect() { - if (GetCurrentThread() == kMainTid) - CurrentThreadContext()->os_id = GetTid(); + if (GetCurrentThreadId() == kMainTid) + GetCurrentThread()->os_id = GetTid(); } ///// Interface to the common LSan module. ///// @@ -79,9 +83,15 @@ void GetThreadExtraStackRangesLocked(tid_t os_id, InternalMmapVector *ranges) {} void GetThreadExtraStackRangesLocked(InternalMmapVector *ranges) {} -void LockThreadRegistry() { thread_registry->Lock(); } +void LockThreads() { + thread_registry->Lock(); + thread_arg_retval->Lock(); +} -void UnlockThreadRegistry() { thread_registry->Unlock(); } +void UnlockThreads() { + thread_arg_retval->Unlock(); + thread_registry->Unlock(); +} ThreadRegistry *GetLsanThreadRegistryLocked() { thread_registry->CheckLocked(); @@ -99,6 +109,10 @@ void GetRunningThreadsLocked(InternalMmapVector *threads) { threads); } +void GetAdditionalThreadContextPtrsLocked(InternalMmapVector *ptrs) { + GetThreadArgRetval().GetAllPtrsLocked(ptrs); +} + } // namespace __lsan namespace __sanitizer { diff --git a/system/lib/compiler-rt/lib/lsan/lsan_thread.h b/system/lib/compiler-rt/lib/lsan/lsan_thread.h index 049c7e2038017..222066ee93cd9 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_thread.h +++ b/system/lib/compiler-rt/lib/lsan/lsan_thread.h @@ -14,6 +14,7 @@ #ifndef LSAN_THREAD_H #define LSAN_THREAD_H +#include "sanitizer_common/sanitizer_thread_arg_retval.h" #include "sanitizer_common/sanitizer_thread_registry.h" namespace __lsan { @@ -21,6 +22,7 @@ namespace __lsan { class ThreadContextLsanBase : public ThreadContextBase { public: explicit ThreadContextLsanBase(int tid); + void OnStarted(void *arg) override; void OnFinished() override; uptr stack_begin() { return stack_begin_; } uptr stack_end() { return stack_end_; } @@ -42,17 +44,21 @@ class ThreadContextLsanBase : public ThreadContextBase { // This subclass of ThreadContextLsanBase is declared in an OS-specific header. class ThreadContext; -void InitializeThreadRegistry(); +void InitializeThreads(); void InitializeMainThread(); ThreadRegistry *GetLsanThreadRegistryLocked(); +ThreadArgRetval &GetThreadArgRetval(); u32 ThreadCreate(u32 tid, bool detached, void *arg = nullptr); void ThreadFinish(); -u32 GetCurrentThread(); -void SetCurrentThread(u32 tid); -ThreadContext *CurrentThreadContext(); +ThreadContextLsanBase *GetCurrentThread(); +inline u32 GetCurrentThreadId() { + ThreadContextLsanBase *ctx = GetCurrentThread(); + return ctx ? ctx->tid : kInvalidTid; +} +void SetCurrentThread(ThreadContextLsanBase *tctx); void EnsureMainThreadIDIsCorrect(); } // namespace __lsan diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp index 25a43a59f0475..03392b61503b0 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.cpp @@ -146,12 +146,10 @@ void *LowLevelAllocator::Allocate(uptr size) { size = RoundUpTo(size, low_level_alloc_min_alignment); if (allocated_end_ - allocated_current_ < (sptr)size) { uptr size_to_allocate = RoundUpTo(size, GetPageSizeCached()); - allocated_current_ = - (char*)MmapOrDie(size_to_allocate, __func__); + allocated_current_ = (char *)MmapOrDie(size_to_allocate, __func__); allocated_end_ = allocated_current_ + size_to_allocate; if (low_level_alloc_callback) { - low_level_alloc_callback((uptr)allocated_current_, - size_to_allocate); + low_level_alloc_callback((uptr)allocated_current_, size_to_allocate); } } CHECK(allocated_end_ - allocated_current_ >= (sptr)size); diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h index 76b936ff5eaa6..0b28f86d14084 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator.h @@ -62,6 +62,13 @@ inline void RandomShuffle(T *a, u32 n, u32 *rand_state) { *rand_state = state; } +struct NoOpMapUnmapCallback { + void OnMap(uptr p, uptr size) const {} + void OnMapSecondary(uptr p, uptr size, uptr user_begin, + uptr user_size) const {} + void OnUnmap(uptr p, uptr size) const {} +}; + #include "sanitizer_allocator_size_class_map.h" #include "sanitizer_allocator_stats.h" #include "sanitizer_allocator_primary64.h" diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h index b76d36dcf5a4e..49940d9b5d505 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_combined.h @@ -29,9 +29,9 @@ class CombinedAllocator { LargeMmapAllocatorPtrArray, typename PrimaryAllocator::AddressSpaceView>; - void InitLinkerInitialized(s32 release_to_os_interval_ms) { - stats_.InitLinkerInitialized(); - primary_.Init(release_to_os_interval_ms); + void InitLinkerInitialized(s32 release_to_os_interval_ms, + uptr heap_start = 0) { + primary_.Init(release_to_os_interval_ms, heap_start); secondary_.InitLinkerInitialized(); } diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h index c1b27563e2fc7..de2b271fb0ed9 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_interface.h @@ -21,8 +21,12 @@ extern "C" { SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_estimated_allocated_size(uptr size); SANITIZER_INTERFACE_ATTRIBUTE int __sanitizer_get_ownership(const void *p); +SANITIZER_INTERFACE_ATTRIBUTE const void *__sanitizer_get_allocated_begin( + const void *p); SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_allocated_size(const void *p); +SANITIZER_INTERFACE_ATTRIBUTE uptr +__sanitizer_get_allocated_size_fast(const void *p); SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_current_allocated_bytes(); SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_heap_size(); SANITIZER_INTERFACE_ATTRIBUTE uptr __sanitizer_get_free_bytes(); diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h index 38994736877ac..62523c7ae187c 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_internal.h @@ -51,7 +51,6 @@ void InternalFree(void *p, InternalAllocatorCache *cache = nullptr); void InternalAllocatorLock(); void InternalAllocatorUnlock(); InternalAllocator *internal_allocator(); - } // namespace __sanitizer #endif // SANITIZER_ALLOCATOR_INTERNAL_H diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h index f2471efced613..52fe3fe3d15bd 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary32.h @@ -353,7 +353,7 @@ class SizeClassAllocator32 { DCHECK_GT(max_count, 0); TransferBatch *b = nullptr; constexpr uptr kShuffleArraySize = 48; - uptr shuffle_array[kShuffleArraySize]; + UNINITIALIZED uptr shuffle_array[kShuffleArraySize]; uptr count = 0; for (uptr i = region; i < region + n_chunks * size; i += size) { shuffle_array[count++] = i; diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h index 66ba71d325dad..fa43ac50c61e4 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_primary64.h @@ -635,8 +635,8 @@ class SizeClassAllocator64 { return kUsingConstantSpaceBeg ? kSpaceBeg : NonConstSpaceBeg; } uptr SpaceEnd() const { return SpaceBeg() + kSpaceSize; } - // kRegionSize must be >= 2^32. - COMPILER_CHECK((kRegionSize) >= (1ULL << (SANITIZER_WORDSIZE / 2))); + // kRegionSize should be able to satisfy the largest size class. + static_assert(kRegionSize >= SizeClassMap::kMaxSize); // kRegionSize must be <= 2^36, see CompactPtrT. COMPILER_CHECK((kRegionSize) <= (1ULL << (SANITIZER_WORDSIZE / 2 + 4))); // Call mmap for user memory with at least this size. diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h index 1576455556044..0607819e7ef7c 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_secondary.h @@ -82,7 +82,7 @@ class LargeMmapAllocator { InitLinkerInitialized(); } - void *Allocate(AllocatorStats *stat, uptr size, uptr alignment) { + void *Allocate(AllocatorStats *stat, const uptr size, uptr alignment) { CHECK(IsPowerOfTwo(alignment)); uptr map_size = RoundUpMapSize(size); if (alignment > page_size_) @@ -99,11 +99,11 @@ class LargeMmapAllocator { if (!map_beg) return nullptr; CHECK(IsAligned(map_beg, page_size_)); - MapUnmapCallback().OnMap(map_beg, map_size); uptr map_end = map_beg + map_size; uptr res = map_beg + page_size_; if (res & (alignment - 1)) // Align. res += alignment - (res & (alignment - 1)); + MapUnmapCallback().OnMapSecondary(map_beg, map_size, res, size); CHECK(IsAligned(res, alignment)); CHECK(IsAligned(res, page_size_)); CHECK_GE(res + size, map_beg); diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_stats.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_stats.h index 6f14e3863c31c..ae4dac9c8c962 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_stats.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_allocator_stats.h @@ -25,19 +25,13 @@ typedef uptr AllocatorStatCounters[AllocatorStatCount]; // Per-thread stats, live in per-thread cache. class AllocatorStats { public: - void Init() { - internal_memset(this, 0, sizeof(*this)); - } - void InitLinkerInitialized() {} - + void Init() { internal_memset(this, 0, sizeof(*this)); } void Add(AllocatorStat i, uptr v) { - v += atomic_load(&stats_[i], memory_order_relaxed); - atomic_store(&stats_[i], v, memory_order_relaxed); + atomic_fetch_add(&stats_[i], v, memory_order_relaxed); } void Sub(AllocatorStat i, uptr v) { - v = atomic_load(&stats_[i], memory_order_relaxed) - v; - atomic_store(&stats_[i], v, memory_order_relaxed); + atomic_fetch_sub(&stats_[i], v, memory_order_relaxed); } void Set(AllocatorStat i, uptr v) { @@ -58,17 +52,13 @@ class AllocatorStats { // Global stats, used for aggregation and querying. class AllocatorGlobalStats : public AllocatorStats { public: - void InitLinkerInitialized() { - next_ = this; - prev_ = this; - } void Init() { internal_memset(this, 0, sizeof(*this)); - InitLinkerInitialized(); } void Register(AllocatorStats *s) { SpinMutexLock l(&mu_); + LazyInit(); s->next_ = next_; s->prev_ = this; next_->prev_ = s; @@ -87,7 +77,7 @@ class AllocatorGlobalStats : public AllocatorStats { internal_memset(s, 0, AllocatorStatCount * sizeof(uptr)); SpinMutexLock l(&mu_); const AllocatorStats *stats = this; - for (;;) { + for (; stats;) { for (int i = 0; i < AllocatorStatCount; i++) s[i] += stats->Get(AllocatorStat(i)); stats = stats->next_; @@ -100,6 +90,13 @@ class AllocatorGlobalStats : public AllocatorStats { } private: + void LazyInit() { + if (!next_) { + next_ = this; + prev_ = this; + } + } + mutable StaticSpinMutex mu_; }; diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_array_ref.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_array_ref.h new file mode 100644 index 0000000000000..28d125383da41 --- /dev/null +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_array_ref.h @@ -0,0 +1,123 @@ +//===-- sanitizer_array_ref.h -----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_ARRAY_REF_H +#define SANITIZER_ARRAY_REF_H + +#include "sanitizer_internal_defs.h" + +namespace __sanitizer { + +/// ArrayRef - Represent a constant reference to an array (0 or more elements +/// consecutively in memory), i.e. a start pointer and a length. It allows +/// various APIs to take consecutive elements easily and conveniently. +/// +/// This class does not own the underlying data, it is expected to be used in +/// situations where the data resides in some other buffer, whose lifetime +/// extends past that of the ArrayRef. For this reason, it is not in general +/// safe to store an ArrayRef. +/// +/// This is intended to be trivially copyable, so it should be passed by +/// value. +template +class ArrayRef { + public: + constexpr ArrayRef() {} + constexpr ArrayRef(const T *begin, const T *end) : begin_(begin), end_(end) { + DCHECK(empty() || begin); + } + constexpr ArrayRef(const T *data, uptr length) + : ArrayRef(data, data + length) {} + template + constexpr ArrayRef(const T (&src)[N]) : ArrayRef(src, src + N) {} + template + constexpr ArrayRef(const C &src) + : ArrayRef(src.data(), src.data() + src.size()) {} + ArrayRef(const T &one_elt) : ArrayRef(&one_elt, &one_elt + 1) {} + + const T *data() const { return empty() ? nullptr : begin_; } + + const T *begin() const { return begin_; } + const T *end() const { return end_; } + + bool empty() const { return begin_ == end_; } + + uptr size() const { return end_ - begin_; } + + /// equals - Check for element-wise equality. + bool equals(ArrayRef rhs) const { + if (size() != rhs.size()) + return false; + auto r = rhs.begin(); + for (auto &l : *this) { + if (!(l == *r)) + return false; + ++r; + } + return true; + } + + /// slice(n, m) - Chop off the first N elements of the array, and keep M + /// elements in the array. + ArrayRef slice(uptr N, uptr M) const { + DCHECK_LE(N + M, size()); + return ArrayRef(data() + N, M); + } + + /// slice(n) - Chop off the first N elements of the array. + ArrayRef slice(uptr N) const { return slice(N, size() - N); } + + /// Drop the first \p N elements of the array. + ArrayRef drop_front(uptr N = 1) const { + DCHECK_GE(size(), N); + return slice(N, size() - N); + } + + /// Drop the last \p N elements of the array. + ArrayRef drop_back(uptr N = 1) const { + DCHECK_GE(size(), N); + return slice(0, size() - N); + } + + /// Return a copy of *this with only the first \p N elements. + ArrayRef take_front(uptr N = 1) const { + if (N >= size()) + return *this; + return drop_back(size() - N); + } + + /// Return a copy of *this with only the last \p N elements. + ArrayRef take_back(uptr N = 1) const { + if (N >= size()) + return *this; + return drop_front(size() - N); + } + + const T &operator[](uptr index) const { + DCHECK_LT(index, size()); + return begin_[index]; + } + + private: + const T *begin_ = nullptr; + const T *end_ = nullptr; +}; + +template +inline bool operator==(ArrayRef lhs, ArrayRef rhs) { + return lhs.equals(rhs); +} + +template +inline bool operator!=(ArrayRef lhs, ArrayRef rhs) { + return !(lhs == rhs); +} + +} // namespace __sanitizer + +#endif // SANITIZER_ARRAY_REF_H diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h index 9ebba91da73ff..3c9bbdc9678b0 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_asm.h @@ -42,13 +42,57 @@ # define CFI_RESTORE(reg) #endif +#if defined(__x86_64__) || defined(__i386__) || defined(__sparc__) +# define ASM_TAIL_CALL jmp +#elif defined(__arm__) || defined(__aarch64__) || defined(__mips__) || \ + defined(__powerpc__) || defined(__loongarch_lp64) +# define ASM_TAIL_CALL b +#elif defined(__s390__) +# define ASM_TAIL_CALL jg +#elif defined(__riscv) +# define ASM_TAIL_CALL tail +#endif + +#if defined(__ELF__) && defined(__x86_64__) || defined(__i386__) || \ + defined(__riscv) +# define ASM_PREEMPTIBLE_SYM(sym) sym@plt +#else +# define ASM_PREEMPTIBLE_SYM(sym) sym +#endif + #if !defined(__APPLE__) # define ASM_HIDDEN(symbol) .hidden symbol # define ASM_TYPE_FUNCTION(symbol) .type symbol, %function # define ASM_SIZE(symbol) .size symbol, .-symbol # define ASM_SYMBOL(symbol) symbol # define ASM_SYMBOL_INTERCEPTOR(symbol) symbol -# define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol +# if defined(__i386__) || defined(__powerpc__) || defined(__s390__) || \ + defined(__sparc__) +// For details, see interception.h +# define ASM_WRAPPER_NAME(symbol) __interceptor_##symbol +# define ASM_TRAMPOLINE_ALIAS(symbol, name) \ + .weak symbol; \ + .set symbol, ASM_WRAPPER_NAME(name) +# define ASM_INTERCEPTOR_TRAMPOLINE(name) +# define ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT 0 +# else // Architecture supports interceptor trampoline +// Keep trampoline implementation in sync with interception/interception.h +# define ASM_WRAPPER_NAME(symbol) ___interceptor_##symbol +# define ASM_TRAMPOLINE_ALIAS(symbol, name) \ + .weak symbol; \ + .set symbol, __interceptor_trampoline_##name +# define ASM_INTERCEPTOR_TRAMPOLINE(name) \ + .weak __interceptor_##name; \ + .set __interceptor_##name, ASM_WRAPPER_NAME(name); \ + .globl __interceptor_trampoline_##name; \ + ASM_TYPE_FUNCTION(__interceptor_trampoline_##name); \ + __interceptor_trampoline_##name: \ + CFI_STARTPROC; \ + ASM_TAIL_CALL ASM_PREEMPTIBLE_SYM(__interceptor_##name); \ + CFI_ENDPROC; \ + ASM_SIZE(__interceptor_trampoline_##name) +# define ASM_INTERCEPTOR_TRAMPOLINE_SUPPORT 1 +# endif // Architecture supports interceptor trampoline #else # define ASM_HIDDEN(symbol) # define ASM_TYPE_FUNCTION(symbol) diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp index 82236453157fa..79b7748b8f6e8 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.cpp @@ -61,6 +61,26 @@ void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type, UNREACHABLE("unable to mmap"); } +void NORETURN ReportMunmapFailureAndDie(void *addr, uptr size, error_t err, + bool raw_report) { + static int recursion_count; + if (raw_report || recursion_count) { + // If raw report is requested or we went into recursion just die. The + // Report() and CHECK calls below may call munmap recursively and fail. + RawWrite("ERROR: Failed to munmap\n"); + Die(); + } + recursion_count++; + Report( + "ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p (error " + "code: %d)\n", + SanitizerToolName, size, size, addr, err); +#if !SANITIZER_GO + DumpProcessMap(); +#endif + UNREACHABLE("unable to unmmap"); +} + typedef bool UptrComparisonFunction(const uptr &a, const uptr &b); typedef bool U32ComparisonFunction(const u32 &a, const u32 &b); diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.h index b462e388c232e..e7e4b8cb506db 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common.h @@ -117,6 +117,7 @@ void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, // unaccessible memory. bool MprotectNoAccess(uptr addr, uptr size); bool MprotectReadOnly(uptr addr, uptr size); +bool MprotectReadWrite(uptr addr, uptr size); void MprotectMallocZones(void *addr, int prot); @@ -211,6 +212,7 @@ class LowLevelAllocator { public: // Requires an external lock. void *Allocate(uptr size); + private: char *allocated_end_; char *allocated_current_; @@ -315,6 +317,8 @@ CheckFailed(const char *file, int line, const char *cond, u64 v1, u64 v2); void NORETURN ReportMmapFailureAndDie(uptr size, const char *mem_type, const char *mmap_type, error_t err, bool raw_report = false); +void NORETURN ReportMunmapFailureAndDie(void *ptr, uptr size, error_t err, + bool raw_report = false); // Returns true if the platform-specific error reported is an OOM error. bool ErrorIsOOM(error_t err); @@ -516,8 +520,8 @@ class InternalMmapVectorNoCtor { return data_[i]; } void push_back(const T &element) { - CHECK_LE(size_, capacity()); - if (size_ == capacity()) { + if (UNLIKELY(size_ >= capacity())) { + CHECK_EQ(size_, capacity()); uptr new_capacity = RoundUpToPowerOfTwo(size_ + 1); Realloc(new_capacity); } @@ -577,7 +581,7 @@ class InternalMmapVectorNoCtor { } private: - void Realloc(uptr new_capacity) { + NOINLINE void Realloc(uptr new_capacity) { CHECK_GT(new_capacity, 0); CHECK_LE(size_, new_capacity); uptr new_capacity_bytes = @@ -793,7 +797,11 @@ inline const char *ModuleArchToString(ModuleArch arch) { return ""; } +#if SANITIZER_APPLE +const uptr kModuleUUIDSize = 16; +#else const uptr kModuleUUIDSize = 32; +#endif const uptr kMaxSegName = 16; // Represents a binary loaded into virtual memory (e.g. this can be an @@ -1076,20 +1084,6 @@ inline u32 GetNumberOfCPUsCached() { return NumberOfCPUsCached; } -template -class ArrayRef { - public: - ArrayRef() {} - ArrayRef(T *begin, T *end) : begin_(begin), end_(end) {} - - T *begin() { return begin_; } - T *end() { return end_; } - - private: - T *begin_ = nullptr; - T *end_ = nullptr; -}; - } // namespace __sanitizer inline void *operator new(__sanitizer::operator_new_size_type size, diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc index e999239549ccc..0e563fa12022a 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors.inc @@ -26,10 +26,8 @@ // COMMON_INTERCEPTOR_SET_PTHREAD_NAME // COMMON_INTERCEPTOR_HANDLE_RECVMSG // COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED -// COMMON_INTERCEPTOR_MEMSET_IMPL -// COMMON_INTERCEPTOR_MEMMOVE_IMPL -// COMMON_INTERCEPTOR_MEMCPY_IMPL // COMMON_INTERCEPTOR_MMAP_IMPL +// COMMON_INTERCEPTOR_MUNMAP_IMPL // COMMON_INTERCEPTOR_COPY_STRING // COMMON_INTERCEPTOR_STRNDUP_IMPL // COMMON_INTERCEPTOR_STRERROR @@ -198,15 +196,6 @@ extern const short *_tolower_tab_; #define wait4 __wait4_time64 #endif -// Platform-specific options. -#if SANITIZER_APPLE -#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0 -#elif SANITIZER_WINDOWS64 -#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0 -#else -#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 1 -#endif // SANITIZER_APPLE - #ifndef COMMON_INTERCEPTOR_INITIALIZE_RANGE #define COMMON_INTERCEPTOR_INITIALIZE_RANGE(p, size) {} #endif @@ -302,53 +291,17 @@ extern const short *_tolower_tab_; COMMON_INTERCEPT_FUNCTION(fn) #endif -#ifndef COMMON_INTERCEPTOR_MEMSET_IMPL -#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \ - { \ - if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \ - return internal_memset(dst, v, size); \ - COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size); \ - if (common_flags()->intercept_intrin) \ - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \ - return REAL(memset)(dst, v, size); \ - } -#endif - -#ifndef COMMON_INTERCEPTOR_MEMMOVE_IMPL -#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size) \ - { \ - if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \ - return internal_memmove(dst, src, size); \ - COMMON_INTERCEPTOR_ENTER(ctx, memmove, dst, src, size); \ - if (common_flags()->intercept_intrin) { \ - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \ - COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \ - } \ - return REAL(memmove)(dst, src, size); \ - } -#endif - -#ifndef COMMON_INTERCEPTOR_MEMCPY_IMPL -#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size) \ - { \ - if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) { \ - return internal_memmove(dst, src, size); \ - } \ - COMMON_INTERCEPTOR_ENTER(ctx, memcpy, dst, src, size); \ - if (common_flags()->intercept_intrin) { \ - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \ - COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \ - } \ - return REAL(memcpy)(dst, src, size); \ - } -#endif - #ifndef COMMON_INTERCEPTOR_MMAP_IMPL #define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \ off) \ { return REAL(mmap)(addr, sz, prot, flags, fd, off); } #endif +#ifndef COMMON_INTERCEPTOR_MUNMAP_IMPL +#define COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, sz) \ + { return REAL(munmap)(addr, sz); } +#endif + #ifndef COMMON_INTERCEPTOR_COPY_STRING #define COMMON_INTERCEPTOR_COPY_STRING(ctx, to, from, size) {} #endif @@ -841,57 +794,6 @@ INTERCEPTOR(char *, strpbrk, const char *s1, const char *s2) { #define INIT_STRPBRK #endif -#if SANITIZER_INTERCEPT_MEMSET -INTERCEPTOR(void *, memset, void *dst, int v, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size); -} - -#define INIT_MEMSET COMMON_INTERCEPT_FUNCTION(memset) -#else -#define INIT_MEMSET -#endif - -#if SANITIZER_INTERCEPT_MEMMOVE -INTERCEPTOR(void *, memmove, void *dst, const void *src, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size); -} - -#define INIT_MEMMOVE COMMON_INTERCEPT_FUNCTION(memmove) -#else -#define INIT_MEMMOVE -#endif - -#if SANITIZER_INTERCEPT_MEMCPY -INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) { - // On OS X, calling internal_memcpy here will cause memory corruptions, - // because memcpy and memmove are actually aliases of the same - // implementation. We need to use internal_memmove here. - // N.B.: If we switch this to internal_ we'll have to use internal_memmove - // due to memcpy being an alias of memmove on OS X. - void *ctx; -#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE - COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size); -#else - COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size); -#endif -} - -#define INIT_MEMCPY \ - do { \ - if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) { \ - COMMON_INTERCEPT_FUNCTION(memcpy); \ - } else { \ - ASSIGN_REAL(memcpy, memmove); \ - } \ - CHECK(REAL(memcpy)); \ - } while (false) - -#else -#define INIT_MEMCPY -#endif - #if SANITIZER_INTERCEPT_MEMCMP DECLARE_WEAK_INTERCEPTOR_HOOK(__sanitizer_weak_hook_memcmp, uptr called_pc, const void *s1, const void *s2, uptr n, @@ -1350,7 +1252,7 @@ INTERCEPTOR(int, prctl, int option, unsigned long arg2, unsigned long arg3, char *name = (char *)arg5; COMMON_INTERCEPTOR_READ_RANGE(ctx, name, internal_strlen(name) + 1); } - int res = REAL(prctl(option, arg2, arg3, arg4, arg5)); + int res = REAL(prctl)(option, arg2, arg3, arg4, arg5); if (option == PR_SET_NAME) { char buff[16]; internal_strncpy(buff, (char *)arg2, 15); @@ -1589,6 +1491,16 @@ VSCANF_INTERCEPTOR_IMPL(__isoc99_vsscanf, false, str, format, ap) INTERCEPTOR(int, __isoc99_vfscanf, void *stream, const char *format, va_list ap) VSCANF_INTERCEPTOR_IMPL(__isoc99_vfscanf, false, stream, format, ap) + +INTERCEPTOR(int, __isoc23_vscanf, const char *format, va_list ap) +VSCANF_INTERCEPTOR_IMPL(__isoc23_vscanf, false, format, ap) + +INTERCEPTOR(int, __isoc23_vsscanf, const char *str, const char *format, + va_list ap) +VSCANF_INTERCEPTOR_IMPL(__isoc23_vsscanf, false, str, format, ap) + +INTERCEPTOR(int, __isoc23_vfscanf, void *stream, const char *format, va_list ap) +VSCANF_INTERCEPTOR_IMPL(__isoc23_vfscanf, false, stream, format, ap) #endif // SANITIZER_INTERCEPT_ISOC99_SCANF INTERCEPTOR(int, scanf, const char *format, ...) @@ -1609,6 +1521,15 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_fscanf, __isoc99_vfscanf, stream, format) INTERCEPTOR(int, __isoc99_sscanf, const char *str, const char *format, ...) FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format) + +INTERCEPTOR(int, __isoc23_scanf, const char *format, ...) +FORMAT_INTERCEPTOR_IMPL(__isoc23_scanf, __isoc23_vscanf, format) + +INTERCEPTOR(int, __isoc23_fscanf, void *stream, const char *format, ...) +FORMAT_INTERCEPTOR_IMPL(__isoc23_fscanf, __isoc23_vfscanf, stream, format) + +INTERCEPTOR(int, __isoc23_sscanf, const char *str, const char *format, ...) +FORMAT_INTERCEPTOR_IMPL(__isoc23_sscanf, __isoc23_vsscanf, str, format) #endif #endif @@ -1632,7 +1553,13 @@ FORMAT_INTERCEPTOR_IMPL(__isoc99_sscanf, __isoc99_vsscanf, str, format) COMMON_INTERCEPT_FUNCTION(__isoc99_fscanf); \ COMMON_INTERCEPT_FUNCTION(__isoc99_vscanf); \ COMMON_INTERCEPT_FUNCTION(__isoc99_vsscanf); \ - COMMON_INTERCEPT_FUNCTION(__isoc99_vfscanf); + COMMON_INTERCEPT_FUNCTION(__isoc99_vfscanf); \ + COMMON_INTERCEPT_FUNCTION(__isoc23_scanf); \ + COMMON_INTERCEPT_FUNCTION(__isoc23_sscanf); \ + COMMON_INTERCEPT_FUNCTION(__isoc23_fscanf); \ + COMMON_INTERCEPT_FUNCTION(__isoc23_vscanf); \ + COMMON_INTERCEPT_FUNCTION(__isoc23_vsscanf); \ + COMMON_INTERCEPT_FUNCTION(__isoc23_vfscanf); #else #define INIT_ISOC99_SCANF #endif @@ -3416,7 +3343,8 @@ INTERCEPTOR(__sanitizer_dirent *, readdir, void *dirp) { // its metadata. See // https://github.com/google/sanitizers/issues/321. __sanitizer_dirent *res = REAL(readdir)(dirp); - if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, res->d_reclen); + if (res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, __sanitizer_dirsiz(res)); return res; } @@ -3431,7 +3359,7 @@ INTERCEPTOR(int, readdir_r, void *dirp, __sanitizer_dirent *entry, if (!res) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); if (*result) - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *result, (*result)->d_reclen); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *result, __sanitizer_dirsiz(*result)); } return res; } @@ -3452,7 +3380,8 @@ INTERCEPTOR(__sanitizer_dirent64 *, readdir64, void *dirp) { // its metadata. See // https://github.com/google/sanitizers/issues/321. __sanitizer_dirent64 *res = REAL(readdir64)(dirp); - if (res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, res->d_reclen); + if (res) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, __sanitizer_dirsiz(res)); return res; } @@ -3467,7 +3396,7 @@ INTERCEPTOR(int, readdir64_r, void *dirp, __sanitizer_dirent64 *entry, if (!res) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, result, sizeof(*result)); if (*result) - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *result, (*result)->d_reclen); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *result, __sanitizer_dirsiz(*result)); } return res; } @@ -3635,30 +3564,26 @@ UNUSED static inline void StrtolFixAndCheck(void *ctx, const char *nptr, (real_endptr - nptr) + 1 : 0); } - #if SANITIZER_INTERCEPT_STRTOIMAX -INTERCEPTOR(INTMAX_T, strtoimax, const char *nptr, char **endptr, int base) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, strtoimax, nptr, endptr, base); - // FIXME: under ASan the call below may write to freed memory and corrupt - // its metadata. See - // https://github.com/google/sanitizers/issues/321. +template +static ALWAYS_INLINE auto StrtoimaxImpl(void *ctx, Fn real, const char *nptr, + char **endptr, int base) + -> decltype(real(nullptr, nullptr, 0)) { char *real_endptr; - INTMAX_T res = REAL(strtoimax)(nptr, &real_endptr, base); + auto res = real(nptr, &real_endptr, base); StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); return res; } +INTERCEPTOR(INTMAX_T, strtoimax, const char *nptr, char **endptr, int base) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, strtoimax, nptr, endptr, base); + return StrtoimaxImpl(ctx, REAL(strtoimax), nptr, endptr, base); +} INTERCEPTOR(UINTMAX_T, strtoumax, const char *nptr, char **endptr, int base) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, strtoumax, nptr, endptr, base); - // FIXME: under ASan the call below may write to freed memory and corrupt - // its metadata. See - // https://github.com/google/sanitizers/issues/321. - char *real_endptr; - UINTMAX_T res = REAL(strtoumax)(nptr, &real_endptr, base); - StrtolFixAndCheck(ctx, nptr, endptr, real_endptr, base); - return res; + return StrtoimaxImpl(ctx, REAL(strtoumax), nptr, endptr, base); } #define INIT_STRTOIMAX \ @@ -3668,6 +3593,25 @@ INTERCEPTOR(UINTMAX_T, strtoumax, const char *nptr, char **endptr, int base) { #define INIT_STRTOIMAX #endif +#if SANITIZER_INTERCEPT_STRTOIMAX && SANITIZER_GLIBC +INTERCEPTOR(INTMAX_T, __isoc23_strtoimax, const char *nptr, char **endptr, int base) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, __isoc23_strtoimax, nptr, endptr, base); + return StrtoimaxImpl(ctx, REAL(__isoc23_strtoimax), nptr, endptr, base); +} +INTERCEPTOR(UINTMAX_T, __isoc23_strtoumax, const char *nptr, char **endptr, int base) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, __isoc23_strtoumax, nptr, endptr, base); + return StrtoimaxImpl(ctx, REAL(__isoc23_strtoumax), nptr, endptr, base); +} + +# define INIT_STRTOIMAX_C23 \ + COMMON_INTERCEPT_FUNCTION(__isoc23_strtoimax); \ + COMMON_INTERCEPT_FUNCTION(__isoc23_strtoumax); +#else +# define INIT_STRTOIMAX_C23 +#endif + #if SANITIZER_INTERCEPT_MBSTOWCS INTERCEPTOR(SIZE_T, mbstowcs, wchar_t *dest, const char *src, SIZE_T len) { void *ctx; @@ -4039,7 +3983,7 @@ static THREADLOCAL scandir_compar_f scandir_compar; static int wrapped_scandir_filter(const struct __sanitizer_dirent *dir) { COMMON_INTERCEPTOR_UNPOISON_PARAM(1); - COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, dir->d_reclen); + COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, __sanitizer_dirsiz(dir)); return scandir_filter(dir); } @@ -4047,9 +3991,9 @@ static int wrapped_scandir_compar(const struct __sanitizer_dirent **a, const struct __sanitizer_dirent **b) { COMMON_INTERCEPTOR_UNPOISON_PARAM(2); COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, sizeof(*a)); - COMMON_INTERCEPTOR_INITIALIZE_RANGE(*a, (*a)->d_reclen); + COMMON_INTERCEPTOR_INITIALIZE_RANGE(*a, __sanitizer_dirsiz(*a)); COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, sizeof(*b)); - COMMON_INTERCEPTOR_INITIALIZE_RANGE(*b, (*b)->d_reclen); + COMMON_INTERCEPTOR_INITIALIZE_RANGE(*b, __sanitizer_dirsiz(*b)); return scandir_compar(a, b); } @@ -4073,7 +4017,7 @@ INTERCEPTOR(int, scandir, char *dirp, __sanitizer_dirent ***namelist, COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *namelist, sizeof(**namelist) * res); for (int i = 0; i < res; ++i) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (*namelist)[i], - (*namelist)[i]->d_reclen); + __sanitizer_dirsiz((*namelist)[i])); } return res; } @@ -4092,7 +4036,7 @@ static THREADLOCAL scandir64_compar_f scandir64_compar; static int wrapped_scandir64_filter(const struct __sanitizer_dirent64 *dir) { COMMON_INTERCEPTOR_UNPOISON_PARAM(1); - COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, dir->d_reclen); + COMMON_INTERCEPTOR_INITIALIZE_RANGE(dir, __sanitizer_dirsiz(dir)); return scandir64_filter(dir); } @@ -4100,9 +4044,9 @@ static int wrapped_scandir64_compar(const struct __sanitizer_dirent64 **a, const struct __sanitizer_dirent64 **b) { COMMON_INTERCEPTOR_UNPOISON_PARAM(2); COMMON_INTERCEPTOR_INITIALIZE_RANGE(a, sizeof(*a)); - COMMON_INTERCEPTOR_INITIALIZE_RANGE(*a, (*a)->d_reclen); + COMMON_INTERCEPTOR_INITIALIZE_RANGE(*a, __sanitizer_dirsiz(*a)); COMMON_INTERCEPTOR_INITIALIZE_RANGE(b, sizeof(*b)); - COMMON_INTERCEPTOR_INITIALIZE_RANGE(*b, (*b)->d_reclen); + COMMON_INTERCEPTOR_INITIALIZE_RANGE(*b, __sanitizer_dirsiz(*b)); return scandir64_compar(a, b); } @@ -4127,7 +4071,7 @@ INTERCEPTOR(int, scandir64, char *dirp, __sanitizer_dirent64 ***namelist, COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *namelist, sizeof(**namelist) * res); for (int i = 0; i < res; ++i) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, (*namelist)[i], - (*namelist)[i]->d_reclen); + __sanitizer_dirsiz((*namelist)[i])); } return res; } @@ -4404,12 +4348,16 @@ INTERCEPTOR(int, pthread_sigmask, int how, __sanitizer_sigset_t *set, INTERCEPTOR(int, backtrace, void **buffer, int size) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, backtrace, buffer, size); - // FIXME: under ASan the call below may write to freed memory and corrupt - // its metadata. See - // https://github.com/google/sanitizers/issues/321. - int res = REAL(backtrace)(buffer, size); - if (res && buffer) + // 'buffer' might be freed memory, hence it is unsafe to directly call + // REAL(backtrace)(buffer, size). Instead, we use our own known-good + // scratch buffer. + void **scratch = (void**)InternalAlloc(sizeof(void*) * size); + int res = REAL(backtrace)(scratch, size); + if (res && buffer) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buffer, res * sizeof(*buffer)); + internal_memcpy(buffer, scratch, res * sizeof(*buffer)); + } + InternalFree(scratch); return res; } @@ -4418,9 +4366,8 @@ INTERCEPTOR(char **, backtrace_symbols, void **buffer, int size) { COMMON_INTERCEPTOR_ENTER(ctx, backtrace_symbols, buffer, size); if (buffer && size) COMMON_INTERCEPTOR_READ_RANGE(ctx, buffer, size * sizeof(*buffer)); - // FIXME: under ASan the call below may write to freed memory and corrupt - // its metadata. See - // https://github.com/google/sanitizers/issues/321. + // The COMMON_INTERCEPTOR_READ_RANGE above ensures that 'buffer' is + // valid for reading. char **res = REAL(backtrace_symbols)(buffer, size); if (res && size) { COMMON_INTERCEPTOR_WRITE_RANGE(ctx, res, size * sizeof(*res)); @@ -4453,7 +4400,7 @@ INTERCEPTOR(void, _exit, int status) { #if SANITIZER_INTERCEPT___LIBC_MUTEX INTERCEPTOR(int, __libc_thr_setcancelstate, int state, int *oldstate) -ALIAS(WRAPPER_NAME(pthread_setcancelstate)); +ALIAS(WRAP(pthread_setcancelstate)); #define INIT___LIBC_THR_SETCANCELSTATE \ COMMON_INTERCEPT_FUNCTION(__libc_thr_setcancelstate) @@ -5484,9 +5431,7 @@ INTERCEPTOR(void *, __tls_get_addr, void *arg) { // On PowerPC, we also need to intercept __tls_get_addr_opt, which has // mostly the same semantics as __tls_get_addr, but its presence enables // some optimizations in linker (which are safe to ignore here). -extern "C" __attribute__((alias("__interceptor___tls_get_addr"), - visibility("default"))) -void *__tls_get_addr_opt(void *arg); +INTERCEPTOR(void *, __tls_get_addr_opt, void *arg) ALIAS(WRAP(__tls_get_addr)); #endif #else // SANITIZER_S390 // On s390, we have to intercept two functions here: @@ -5520,21 +5465,20 @@ INTERCEPTOR(uptr, __tls_get_addr_internal, void *arg) { #if SANITIZER_S390 && \ (SANITIZER_INTERCEPT_TLS_GET_ADDR || SANITIZER_INTERCEPT_TLS_GET_OFFSET) -extern "C" uptr __tls_get_offset(void *arg); -extern "C" uptr __interceptor___tls_get_offset(void *arg); // We need a hidden symbol aliasing the above, so that we can jump // directly to it from the assembly below. -extern "C" __attribute__((alias("__interceptor___tls_get_addr_internal"), - visibility("hidden"))) -uptr __tls_get_addr_hidden(void *arg); +extern "C" __attribute__((visibility("hidden"))) uptr __tls_get_addr_hidden( + void *arg) ALIAS(WRAP(__tls_get_addr_internal)); +extern "C" uptr __tls_get_offset(void *arg); +extern "C" uptr TRAMPOLINE(__tls_get_offset)(void *arg); +extern "C" uptr WRAP(__tls_get_offset)(void *arg); // Now carefully intercept __tls_get_offset. asm( ".text\n" // The __intercept_ version has to exist, so that gen_dynamic_list.py // exports our symbol. ".weak __tls_get_offset\n" - ".type __tls_get_offset, @function\n" - "__tls_get_offset:\n" + ".set __tls_get_offset, __interceptor___tls_get_offset\n" ".global __interceptor___tls_get_offset\n" ".type __interceptor___tls_get_offset, @function\n" "__interceptor___tls_get_offset:\n" @@ -5763,8 +5707,10 @@ INTERCEPTOR(int, capget, void *hdrp, void *datap) { // its metadata. See // https://github.com/google/sanitizers/issues/321. int res = REAL(capget)(hdrp, datap); - if (res == 0 && datap) - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, datap, __user_cap_data_struct_sz); + if (res == 0 && datap) { + unsigned datasz = __user_cap_data_struct_sz(hdrp); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, datap, datasz); + } // We can also return -1 and write to hdrp->version if the version passed in // hdrp->version is unsupported. But that's not a trivial condition to check, // and anyway COMMON_INTERCEPTOR_READ_RANGE protects us to some extent. @@ -5775,8 +5721,10 @@ INTERCEPTOR(int, capset, void *hdrp, const void *datap) { COMMON_INTERCEPTOR_ENTER(ctx, capset, hdrp, datap); if (hdrp) COMMON_INTERCEPTOR_READ_RANGE(ctx, hdrp, __user_cap_header_struct_sz); - if (datap) - COMMON_INTERCEPTOR_READ_RANGE(ctx, datap, __user_cap_data_struct_sz); + if (datap) { + unsigned datasz = __user_cap_data_struct_sz(hdrp); + COMMON_INTERCEPTOR_READ_RANGE(ctx, datap, datasz); + } return REAL(capset)(hdrp, datap); } #define INIT_CAPGET \ @@ -5786,105 +5734,6 @@ INTERCEPTOR(int, capset, void *hdrp, const void *datap) { #define INIT_CAPGET #endif -#if SANITIZER_INTERCEPT_AEABI_MEM -INTERCEPTOR(void *, __aeabi_memmove, void *to, const void *from, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size); -} - -INTERCEPTOR(void *, __aeabi_memmove4, void *to, const void *from, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size); -} - -INTERCEPTOR(void *, __aeabi_memmove8, void *to, const void *from, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size); -} - -INTERCEPTOR(void *, __aeabi_memcpy, void *to, const void *from, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size); -} - -INTERCEPTOR(void *, __aeabi_memcpy4, void *to, const void *from, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size); -} - -INTERCEPTOR(void *, __aeabi_memcpy8, void *to, const void *from, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size); -} - -// Note the argument order. -INTERCEPTOR(void *, __aeabi_memset, void *block, uptr size, int c) { - void *ctx; - COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size); -} - -INTERCEPTOR(void *, __aeabi_memset4, void *block, uptr size, int c) { - void *ctx; - COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size); -} - -INTERCEPTOR(void *, __aeabi_memset8, void *block, uptr size, int c) { - void *ctx; - COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size); -} - -INTERCEPTOR(void *, __aeabi_memclr, void *block, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); -} - -INTERCEPTOR(void *, __aeabi_memclr4, void *block, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); -} - -INTERCEPTOR(void *, __aeabi_memclr8, void *block, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); -} - -#define INIT_AEABI_MEM \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memmove); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memmove4); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memmove8); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy4); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy8); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memset); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memset4); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memset8); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memclr); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memclr4); \ - COMMON_INTERCEPT_FUNCTION(__aeabi_memclr8); -#else -#define INIT_AEABI_MEM -#endif // SANITIZER_INTERCEPT_AEABI_MEM - -#if SANITIZER_INTERCEPT___BZERO -INTERCEPTOR(void *, __bzero, void *block, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); -} -#define INIT___BZERO COMMON_INTERCEPT_FUNCTION(__bzero); -#else -#define INIT___BZERO -#endif // SANITIZER_INTERCEPT___BZERO - -#if SANITIZER_INTERCEPT_BZERO -INTERCEPTOR(void *, bzero, void *block, uptr size) { - void *ctx; - COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); -} -#define INIT_BZERO COMMON_INTERCEPT_FUNCTION(bzero); -#else -#define INIT_BZERO -#endif // SANITIZER_INTERCEPT_BZERO - #if SANITIZER_INTERCEPT_FTIME INTERCEPTOR(int, ftime, __sanitizer_timeb *tp) { void *ctx; @@ -7169,6 +7018,7 @@ INTERCEPTOR(int, mprobe, void *ptr) { } #endif +#if SANITIZER_INTERCEPT_WCSLEN INTERCEPTOR(SIZE_T, wcslen, const wchar_t *s) { void *ctx; COMMON_INTERCEPTOR_ENTER(ctx, wcslen, s); @@ -7187,6 +7037,9 @@ INTERCEPTOR(SIZE_T, wcsnlen, const wchar_t *s, SIZE_T n) { #define INIT_WCSLEN \ COMMON_INTERCEPT_FUNCTION(wcslen); \ COMMON_INTERCEPT_FUNCTION(wcsnlen); +#else +#define INIT_WCSLEN +#endif #if SANITIZER_INTERCEPT_WCSCAT INTERCEPTOR(wchar_t *, wcscat, wchar_t *dst, const wchar_t *src) { @@ -7595,6 +7448,14 @@ INTERCEPTOR(void *, mmap, void *addr, SIZE_T sz, int prot, int flags, int fd, COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, off); } +INTERCEPTOR(int, munmap, void *addr, SIZE_T sz) { + void *ctx; + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) + return (int)internal_munmap(addr, sz); + COMMON_INTERCEPTOR_ENTER(ctx, munmap, addr, sz); + COMMON_INTERCEPTOR_MUNMAP_IMPL(ctx, addr, sz); +} + INTERCEPTOR(int, mprotect, void *addr, SIZE_T sz, int prot) { void *ctx; if (common_flags()->detect_write_exec) @@ -7607,6 +7468,7 @@ INTERCEPTOR(int, mprotect, void *addr, SIZE_T sz, int prot) { } #define INIT_MMAP \ COMMON_INTERCEPT_FUNCTION(mmap); \ + COMMON_INTERCEPT_FUNCTION(munmap); \ COMMON_INTERCEPT_FUNCTION(mprotect); #else #define INIT_MMAP @@ -7744,8 +7606,7 @@ static void write_protoent(void *ctx, struct __sanitizer_protoent *p) { for (char **pp = p->p_aliases; *pp; ++pp, ++pp_size) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *pp, internal_strlen(*pp) + 1); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases, - pp_size * sizeof(char **)); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, p->p_aliases, pp_size * sizeof(char *)); } INTERCEPTOR(struct __sanitizer_protoent *, getprotoent) { @@ -7851,8 +7712,7 @@ INTERCEPTOR(struct __sanitizer_netent *, getnetent) { for (char **nn = n->n_aliases; *nn; ++nn, ++nn_size) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *nn, internal_strlen(*nn) + 1); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases, - nn_size * sizeof(char **)); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases, nn_size * sizeof(char *)); } return n; } @@ -7873,8 +7733,7 @@ INTERCEPTOR(struct __sanitizer_netent *, getnetbyname, const char *name) { for (char **nn = n->n_aliases; *nn; ++nn, ++nn_size) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *nn, internal_strlen(*nn) + 1); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases, - nn_size * sizeof(char **)); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases, nn_size * sizeof(char *)); } return n; } @@ -7893,8 +7752,7 @@ INTERCEPTOR(struct __sanitizer_netent *, getnetbyaddr, u32 net, int type) { for (char **nn = n->n_aliases; *nn; ++nn, ++nn_size) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, *nn, internal_strlen(*nn) + 1); - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases, - nn_size * sizeof(char **)); + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, n->n_aliases, nn_size * sizeof(char *)); } return n; } @@ -10086,41 +9944,6 @@ INTERCEPTOR(SSIZE_T, getrandom, void *buf, SIZE_T buflen, unsigned int flags) { #define INIT_GETRANDOM #endif -#if SANITIZER_INTERCEPT_CRYPT -INTERCEPTOR(char *, crypt, char *key, char *salt) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, crypt, key, salt); - COMMON_INTERCEPTOR_READ_RANGE(ctx, key, internal_strlen(key) + 1); - COMMON_INTERCEPTOR_READ_RANGE(ctx, salt, internal_strlen(salt) + 1); - char *res = REAL(crypt)(key, salt); - if (res != nullptr) - COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1); - return res; -} -#define INIT_CRYPT COMMON_INTERCEPT_FUNCTION(crypt); -#else -#define INIT_CRYPT -#endif - -#if SANITIZER_INTERCEPT_CRYPT_R -INTERCEPTOR(char *, crypt_r, char *key, char *salt, void *data) { - void *ctx; - COMMON_INTERCEPTOR_ENTER(ctx, crypt_r, key, salt, data); - COMMON_INTERCEPTOR_READ_RANGE(ctx, key, internal_strlen(key) + 1); - COMMON_INTERCEPTOR_READ_RANGE(ctx, salt, internal_strlen(salt) + 1); - char *res = REAL(crypt_r)(key, salt, data); - if (res != nullptr) { - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, data, - __sanitizer::struct_crypt_data_sz); - COMMON_INTERCEPTOR_INITIALIZE_RANGE(res, internal_strlen(res) + 1); - } - return res; -} -#define INIT_CRYPT_R COMMON_INTERCEPT_FUNCTION(crypt_r); -#else -#define INIT_CRYPT_R -#endif - #if SANITIZER_INTERCEPT_GETENTROPY INTERCEPTOR(int, getentropy, void *buf, SIZE_T buflen) { void *ctx; @@ -10371,14 +10194,52 @@ INTERCEPTOR(void, hexdump, const void *ptr, int length, const char *header, int #define INIT_HEXDUMP #endif +#if SANITIZER_INTERCEPT_ARGP_PARSE +INTERCEPTOR(int, argp_parse, const struct argp *argp, int argc, char **argv, + unsigned flags, int *arg_index, void *input) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, argp_parse, argp, argc, argv, flags, arg_index, + input); + for (int i = 0; i < argc; i++) + COMMON_INTERCEPTOR_READ_RANGE(ctx, argv[i], internal_strlen(argv[i]) + 1); + int res = REAL(argp_parse)(argp, argc, argv, flags, arg_index, input); + if (!res && arg_index) + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, arg_index, sizeof(int)); + return res; +} + +#define INIT_ARGP_PARSE COMMON_INTERCEPT_FUNCTION(argp_parse); +#else +#define INIT_ARGP_PARSE +#endif + +#if SANITIZER_INTERCEPT_CPUSET_GETAFFINITY +INTERCEPTOR(int, cpuset_getaffinity, int level, int which, __int64_t id, SIZE_T cpusetsize, __sanitizer_cpuset_t *mask) { + void *ctx; + COMMON_INTERCEPTOR_ENTER(ctx, cpuset_getaffinity, level, which, id, cpusetsize, mask); + int res = REAL(cpuset_getaffinity)(level, which, id, cpusetsize, mask); + if (mask && !res) COMMON_INTERCEPTOR_WRITE_RANGE(ctx, mask, cpusetsize); + return res; +} +#define INIT_CPUSET_GETAFFINITY COMMON_INTERCEPT_FUNCTION(cpuset_getaffinity); +#else +#define INIT_CPUSET_GETAFFINITY +#endif + #include "sanitizer_common_interceptors_netbsd_compat.inc" +namespace __sanitizer { +void InitializeMemintrinsicInterceptors(); +} // namespace __sanitizer + static void InitializeCommonInterceptors() { #if SI_POSIX static u64 metadata_mem[sizeof(MetadataHashMap) / sizeof(u64) + 1]; interceptor_metadata_map = new ((void *)&metadata_mem) MetadataHashMap(); #endif + __sanitizer::InitializeMemintrinsicInterceptors(); + INIT_MMAP; INIT_MMAP64; INIT_TEXTDOMAIN; @@ -10400,9 +10261,6 @@ static void InitializeCommonInterceptors() { INIT_STRPBRK; INIT_STRXFRM; INIT___STRXFRM_L; - INIT_MEMSET; - INIT_MEMMOVE; - INIT_MEMCPY; INIT_MEMCHR; INIT_MEMCMP; INIT_BCMP; @@ -10486,6 +10344,7 @@ static void InitializeCommonInterceptors() { INIT_GETCWD; INIT_GET_CURRENT_DIR_NAME; INIT_STRTOIMAX; + INIT_STRTOIMAX_C23; INIT_MBSTOWCS; INIT_MBSNRTOWCS; INIT_WCSTOMBS; @@ -10574,9 +10433,6 @@ static void InitializeCommonInterceptors() { INIT_GETIFADDRS; INIT_IF_INDEXTONAME; INIT_CAPGET; - INIT_AEABI_MEM; - INIT___BZERO; - INIT_BZERO; INIT_FTIME; INIT_XDR; INIT_XDRREC_LINUX; @@ -10679,8 +10535,6 @@ static void InitializeCommonInterceptors() { INIT_GETUSERSHELL; INIT_SL_INIT; INIT_GETRANDOM; - INIT_CRYPT; - INIT_CRYPT_R; INIT_GETENTROPY; INIT_QSORT; INIT_QSORT_R; @@ -10690,6 +10544,8 @@ static void InitializeCommonInterceptors() { INIT_UNAME; INIT___XUNAME; INIT_HEXDUMP; + INIT_ARGP_PARSE; + INIT_CPUSET_GETAFFINITY; INIT___PRINTF_CHK; } diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_format.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_format.inc index 220abb89c3beb..24485900644b3 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_format.inc +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_format.inc @@ -340,11 +340,19 @@ static void scanf_common(void *ctx, int n_inputs, bool allowGnuMalloc, size = 0; } COMMON_INTERCEPTOR_WRITE_RANGE(ctx, argp, size); - // For %ms/%mc, write the allocated output buffer as well. + // For %mc/%mC/%ms/%m[/%mS, write the allocated output buffer as well. if (dir.allocate) { - char *buf = *(char **)argp; - if (buf) - COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, internal_strlen(buf) + 1); + if (char *buf = *(char **)argp) { + if (dir.convSpecifier == 'c') + size = 1; + else if (dir.convSpecifier == 'C') + size = sizeof(wchar_t); + else if (dir.convSpecifier == 'S') + size = (internal_wcslen((wchar_t *)buf) + 1) * sizeof(wchar_t); + else // 's' or '[' + size = internal_strlen(buf) + 1; + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, buf, size); + } } } } diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc new file mode 100644 index 0000000000000..52e489d02cda8 --- /dev/null +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interceptors_memintrinsics.inc @@ -0,0 +1,244 @@ +//===-- sanitizer_common_interceptors_memintrinsics.inc ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Memintrinsic function interceptors for tools like AddressSanitizer, +// ThreadSanitizer, MemorySanitizer, etc. +// +// These interceptors are part of the common interceptors, but separated out so +// that implementations may add them, if necessary, to a separate source file +// that should define SANITIZER_COMMON_NO_REDEFINE_BUILTINS at the top. +// +// This file should be included into the tool's memintrinsic interceptor file, +// which has to define its own macros: +// COMMON_INTERCEPTOR_ENTER +// COMMON_INTERCEPTOR_READ_RANGE +// COMMON_INTERCEPTOR_WRITE_RANGE +// COMMON_INTERCEPTOR_MEMSET_IMPL +// COMMON_INTERCEPTOR_MEMMOVE_IMPL +// COMMON_INTERCEPTOR_MEMCPY_IMPL +// COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED +//===----------------------------------------------------------------------===// + +#ifdef SANITIZER_REDEFINE_BUILTINS_H +#error "Define SANITIZER_COMMON_NO_REDEFINE_BUILTINS in .cpp file" +#endif + +#include "interception/interception.h" +#include "sanitizer_platform_interceptors.h" + +// Platform-specific options. +#if SANITIZER_APPLE +#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0 +#elif SANITIZER_WINDOWS64 +#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 0 +#else +#define PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE 1 +#endif // SANITIZER_APPLE + +#ifndef COMMON_INTERCEPTOR_MEMSET_IMPL +#define COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size) \ + { \ + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \ + return internal_memset(dst, v, size); \ + COMMON_INTERCEPTOR_ENTER(ctx, memset, dst, v, size); \ + if (common_flags()->intercept_intrin) \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \ + return REAL(memset)(dst, v, size); \ + } +#endif + +#ifndef COMMON_INTERCEPTOR_MEMMOVE_IMPL +#define COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size) \ + { \ + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) \ + return internal_memmove(dst, src, size); \ + COMMON_INTERCEPTOR_ENTER(ctx, memmove, dst, src, size); \ + if (common_flags()->intercept_intrin) { \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \ + } \ + return REAL(memmove)(dst, src, size); \ + } +#endif + +#ifndef COMMON_INTERCEPTOR_MEMCPY_IMPL +#define COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size) \ + { \ + if (COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED) { \ + return internal_memmove(dst, src, size); \ + } \ + COMMON_INTERCEPTOR_ENTER(ctx, memcpy, dst, src, size); \ + if (common_flags()->intercept_intrin) { \ + COMMON_INTERCEPTOR_WRITE_RANGE(ctx, dst, size); \ + COMMON_INTERCEPTOR_READ_RANGE(ctx, src, size); \ + } \ + return REAL(memcpy)(dst, src, size); \ + } +#endif + +#if SANITIZER_INTERCEPT_MEMSET +INTERCEPTOR(void *, memset, void *dst, int v, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, dst, v, size); +} + +#define INIT_MEMSET COMMON_INTERCEPT_FUNCTION(memset) +#else +#define INIT_MEMSET +#endif + +#if SANITIZER_INTERCEPT_MEMMOVE +INTERCEPTOR(void *, memmove, void *dst, const void *src, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size); +} + +#define INIT_MEMMOVE COMMON_INTERCEPT_FUNCTION(memmove) +#else +#define INIT_MEMMOVE +#endif + +#if SANITIZER_INTERCEPT_MEMCPY +INTERCEPTOR(void *, memcpy, void *dst, const void *src, uptr size) { + // On OS X, calling internal_memcpy here will cause memory corruptions, + // because memcpy and memmove are actually aliases of the same + // implementation. We need to use internal_memmove here. + // N.B.: If we switch this to internal_ we'll have to use internal_memmove + // due to memcpy being an alias of memmove on OS X. + void *ctx; +#if PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE + COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, dst, src, size); +#else + COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, dst, src, size); +#endif +} + +#define INIT_MEMCPY \ + do { \ + if (PLATFORM_HAS_DIFFERENT_MEMCPY_AND_MEMMOVE) { \ + COMMON_INTERCEPT_FUNCTION(memcpy); \ + } else { \ + ASSIGN_REAL(memcpy, memmove); \ + } \ + CHECK(REAL(memcpy)); \ + } while (false) + +#else +#define INIT_MEMCPY +#endif + +#if SANITIZER_INTERCEPT_AEABI_MEM +INTERCEPTOR(void *, __aeabi_memmove, void *to, const void *from, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size); +} + +INTERCEPTOR(void *, __aeabi_memmove4, void *to, const void *from, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size); +} + +INTERCEPTOR(void *, __aeabi_memmove8, void *to, const void *from, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMMOVE_IMPL(ctx, to, from, size); +} + +INTERCEPTOR(void *, __aeabi_memcpy, void *to, const void *from, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size); +} + +INTERCEPTOR(void *, __aeabi_memcpy4, void *to, const void *from, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size); +} + +INTERCEPTOR(void *, __aeabi_memcpy8, void *to, const void *from, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMCPY_IMPL(ctx, to, from, size); +} + +// Note the argument order. +INTERCEPTOR(void *, __aeabi_memset, void *block, uptr size, int c) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size); +} + +INTERCEPTOR(void *, __aeabi_memset4, void *block, uptr size, int c) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size); +} + +INTERCEPTOR(void *, __aeabi_memset8, void *block, uptr size, int c) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, c, size); +} + +INTERCEPTOR(void *, __aeabi_memclr, void *block, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); +} + +INTERCEPTOR(void *, __aeabi_memclr4, void *block, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); +} + +INTERCEPTOR(void *, __aeabi_memclr8, void *block, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); +} + +#define INIT_AEABI_MEM \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memmove); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memmove4); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memmove8); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy4); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memcpy8); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memset); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memset4); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memset8); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memclr); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memclr4); \ + COMMON_INTERCEPT_FUNCTION(__aeabi_memclr8); +#else +#define INIT_AEABI_MEM +#endif // SANITIZER_INTERCEPT_AEABI_MEM + +#if SANITIZER_INTERCEPT___BZERO +INTERCEPTOR(void *, __bzero, void *block, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); +} +#define INIT___BZERO COMMON_INTERCEPT_FUNCTION(__bzero); +#else +#define INIT___BZERO +#endif // SANITIZER_INTERCEPT___BZERO + +#if SANITIZER_INTERCEPT_BZERO +INTERCEPTOR(void *, bzero, void *block, uptr size) { + void *ctx; + COMMON_INTERCEPTOR_MEMSET_IMPL(ctx, block, 0, size); +} +#define INIT_BZERO COMMON_INTERCEPT_FUNCTION(bzero); +#else +#define INIT_BZERO +#endif // SANITIZER_INTERCEPT_BZERO + +namespace __sanitizer { +// This does not need to be called if InitializeCommonInterceptors() is called. +void InitializeMemintrinsicInterceptors() { + INIT_MEMSET; + INIT_MEMMOVE; + INIT_MEMCPY; + INIT_AEABI_MEM; + INIT___BZERO; + INIT_BZERO; +} +} // namespace __sanitizer diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc index 958f071e7b5f7..557207fe62ac6 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_interface.inc @@ -32,7 +32,9 @@ INTERFACE_FUNCTION(__sanitizer_get_module_and_offset_for_pc) INTERFACE_FUNCTION(__sanitizer_symbolize_global) INTERFACE_FUNCTION(__sanitizer_symbolize_pc) // Allocator interface. +INTERFACE_FUNCTION(__sanitizer_get_allocated_begin) INTERFACE_FUNCTION(__sanitizer_get_allocated_size) +INTERFACE_FUNCTION(__sanitizer_get_allocated_size_fast) INTERFACE_FUNCTION(__sanitizer_get_current_allocated_bytes) INTERFACE_FUNCTION(__sanitizer_get_estimated_allocated_size) INTERFACE_FUNCTION(__sanitizer_get_free_bytes) @@ -44,3 +46,7 @@ INTERFACE_FUNCTION(__sanitizer_purge_allocator) INTERFACE_FUNCTION(__sanitizer_print_memory_profile) INTERFACE_WEAK_FUNCTION(__sanitizer_free_hook) INTERFACE_WEAK_FUNCTION(__sanitizer_malloc_hook) +// Memintrinsic functions. +INTERFACE_FUNCTION(__sanitizer_internal_memcpy) +INTERFACE_FUNCTION(__sanitizer_internal_memmove) +INTERFACE_FUNCTION(__sanitizer_internal_memset) diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp index 4c565c5a361b0..85b89866f8fcd 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_libcdep.cpp @@ -67,6 +67,8 @@ void *BackgroundThread(void *arg) { } else if (soft_rss_limit_mb >= current_rss_mb && reached_soft_rss_limit) { reached_soft_rss_limit = false; + Report("%s: soft rss limit unexhausted (%zdMb vs %zdMb)\n", + SanitizerToolName, soft_rss_limit_mb, current_rss_mb); SetRssLimitExceeded(false); } } diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc index 9d7518ac9476c..c10943b3e4879 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_common_syscalls.inc @@ -417,14 +417,14 @@ PRE_SYSCALL(capget)(void *header, void *dataptr) { POST_SYSCALL(capget)(long res, void *header, void *dataptr) { if (res >= 0) if (dataptr) - POST_WRITE(dataptr, __user_cap_data_struct_sz); + POST_WRITE(dataptr, __user_cap_data_struct_sz(header)); } PRE_SYSCALL(capset)(void *header, const void *data) { if (header) PRE_READ(header, __user_cap_header_struct_sz); if (data) - PRE_READ(data, __user_cap_data_struct_sz); + PRE_READ(data, __user_cap_data_struct_sz(header)); } POST_SYSCALL(capset)(long res, void *header, const void *data) {} @@ -1374,9 +1374,8 @@ PRE_SYSCALL(io_setup)(long nr_reqs, void **ctx) { } POST_SYSCALL(io_setup)(long res, long nr_reqs, void **ctx) { - if (res >= 0) { - if (ctx) - POST_WRITE(ctx, sizeof(*ctx)); + if (res >= 0 && ctx) { + POST_WRITE(ctx, sizeof(*ctx)); // (*ctx) is actually a pointer to a kernel mapped page, and there are // people out there who are crazy enough to peek into that page's 32-byte // header. @@ -2136,7 +2135,7 @@ PRE_SYSCALL(epoll_pwait2) const sanitizer_kernel_timespec *timeout, const kernel_sigset_t *sigmask, long sigsetsize) { if (timeout) - PRE_READ(timeout, sizeof(timeout)); + PRE_READ(timeout, sizeof(*timeout)); if (sigmask) PRE_READ(sigmask, sigsetsize); } diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp index 956b48e0b434b..ce4326967180d 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_coverage_libcdep_new.cpp @@ -282,7 +282,14 @@ SANITIZER_INTERFACE_WEAK_DEF(void, __sanitizer_cov_pcs_init, const uptr* beg, // Weak definition for code instrumented with -fsanitize-coverage=stack-depth // and later linked with code containing a strong definition. // E.g., -fsanitize=fuzzer-no-link +// FIXME: Update Apple deployment target so that thread_local is always +// supported, and remove the #if. +// FIXME: Figure out how this should work on Windows, exported thread_local +// symbols are not supported: +// "data with thread storage duration may not have dll interface" +#if !SANITIZER_APPLE && !SANITIZER_WINDOWS SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE -SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE uptr __sancov_lowest_stack; +thread_local uptr __sancov_lowest_stack; +#endif #endif // !SANITIZER_FUCHSIA diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_file.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_file.h index 810c1e452f611..9459c6b00accc 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_file.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_file.h @@ -15,6 +15,7 @@ #ifndef SANITIZER_FILE_H #define SANITIZER_FILE_H +#include "sanitizer_common.h" #include "sanitizer_internal_defs.h" #include "sanitizer_libc.h" #include "sanitizer_mutex.h" diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.cpp index 9e274268bf2a6..c620da7f220af 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.cpp @@ -13,9 +13,9 @@ #include "sanitizer_flag_parser.h" #include "sanitizer_common.h" -#include "sanitizer_libc.h" -#include "sanitizer_flags.h" #include "sanitizer_flag_parser.h" +#include "sanitizer_flags.h" +#include "sanitizer_libc.h" namespace __sanitizer { diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h index 3ccc6a6fa5377..ae49294dde95c 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flag_parser.h @@ -13,9 +13,9 @@ #ifndef SANITIZER_FLAG_REGISTRY_H #define SANITIZER_FLAG_REGISTRY_H +#include "sanitizer_common.h" #include "sanitizer_internal_defs.h" #include "sanitizer_libc.h" -#include "sanitizer_common.h" namespace __sanitizer { diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flat_map.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flat_map.h index 05fb554d20c14..8bb8304910c73 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flat_map.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_flat_map.h @@ -21,12 +21,6 @@ namespace __sanitizer { -// Call these callbacks on mmap/munmap. -struct NoOpMapUnmapCallback { - void OnMap(uptr p, uptr size) const {} - void OnUnmap(uptr p, uptr size) const {} -}; - // Maps integers in rage [0, kSize) to values. template @@ -62,8 +56,7 @@ class FlatMap { // Each value is initially zero and can be set to something else only once. // Setting and getting values from multiple threads is safe w/o extra locking. template + typename AddressSpaceViewTy = LocalAddressSpaceView> class TwoLevelMap { static_assert(IsPowerOfTwo(kSize2), "Use a power of two for performance."); @@ -79,7 +72,6 @@ class TwoLevelMap { T *p = Get(i); if (!p) continue; - MapUnmapCallback().OnUnmap(reinterpret_cast(p), MmapSize()); UnmapOrDie(p, kSize2); } Init(); @@ -149,7 +141,6 @@ class TwoLevelMap { T *res = Get(idx); if (!res) { res = reinterpret_cast(MmapOrDie(MmapSize(), "TwoLevelMap")); - MapUnmapCallback().OnMap(reinterpret_cast(res), kSize2); atomic_store(&map1_[idx], reinterpret_cast(res), memory_order_release); } @@ -164,10 +155,8 @@ template using FlatByteMap = FlatMap; template -using TwoLevelByteMap = - TwoLevelMap; + typename AddressSpaceViewTy = LocalAddressSpaceView> +using TwoLevelByteMap = TwoLevelMap; } // namespace __sanitizer #endif diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp index a92e84cb8ecf7..1e25265c00a20 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_fuchsia.cpp @@ -285,6 +285,12 @@ bool MprotectReadOnly(uptr addr, uptr size) { ZX_OK; } +bool MprotectReadWrite(uptr addr, uptr size) { + return _zx_vmar_protect(_zx_vmar_root_self(), + ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, addr, + size) == ZX_OK; +} + void *MmapAlignedOrDieOnFatalError(uptr size, uptr alignment, const char *mem_type) { CHECK_GE(size, GetPageSize()); diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc index 9683b97ab91d6..16b2a10d8b069 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_interceptors_ioctl_netbsd.inc @@ -1267,8 +1267,6 @@ static void ioctl_table_fill() { _(TIOCGFLAGS, WRITE, sizeof(int)); _(TIOCSFLAGS, READ, sizeof(int)); _(TIOCDCDTIMESTAMP, WRITE, struct_timeval_sz); - _(TIOCRCVFRAME, READ, sizeof(uptr)); - _(TIOCXMTFRAME, READ, sizeof(uptr)); _(TIOCPTMGET, WRITE, struct_ptmget_sz); _(TIOCGRANTPT, NONE, 0); _(TIOCPTSNAME, WRITE, struct_ptmget_sz); diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h index b9090f47beb7b..63ee6e39f664b 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_internal_defs.h @@ -13,6 +13,7 @@ #define SANITIZER_DEFS_H #include "sanitizer_platform.h" +#include "sanitizer_redefine_builtins.h" #ifndef SANITIZER_DEBUG # define SANITIZER_DEBUG 0 @@ -37,15 +38,6 @@ # define SANITIZER_WEAK_ATTRIBUTE __attribute__((weak)) #endif -// TLS is handled differently on different platforms -#if SANITIZER_LINUX || SANITIZER_NETBSD || \ - SANITIZER_FREEBSD -# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE \ - __attribute__((tls_model("initial-exec"))) thread_local -#else -# define SANITIZER_TLS_INITIAL_EXEC_ATTRIBUTE -#endif - //--------------------------- WEAK FUNCTIONS ---------------------------------// // When working with weak functions, to simplify the code and make it more // portable, when possible define a default implementation using this macro: @@ -226,7 +218,7 @@ typedef u64 tid_t; # define WARN_UNUSED_RESULT #else // _MSC_VER # define ALWAYS_INLINE inline __attribute__((always_inline)) -# define ALIAS(x) __attribute__((alias(x))) +# define ALIAS(x) __attribute__((alias(SANITIZER_STRINGIFY(x)))) // Please only use the ALIGNED macro before the type. // Using ALIGNED after the variable declaration is not portable! # define ALIGNED(x) __attribute__((aligned(x))) @@ -267,6 +259,12 @@ typedef u64 tid_t; # define FALLTHROUGH #endif +#if __has_attribute(uninitialized) +# define UNINITIALIZED __attribute__((uninitialized)) +#else +# define UNINITIALIZED +#endif + // Unaligned versions of basic types. typedef ALIGNED(1) u16 uu16; typedef ALIGNED(1) u32 uu32; diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cpp index d3076f0da4891..4a6fa5e8dbacb 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.cpp @@ -10,6 +10,9 @@ // run-time libraries. See sanitizer_libc.h for details. //===----------------------------------------------------------------------===// +// Do not redefine builtins; this file is defining the builtin replacements. +#define SANITIZER_COMMON_NO_REDEFINE_BUILTINS + #include "sanitizer_allocator_internal.h" #include "sanitizer_common.h" #include "sanitizer_libc.h" @@ -46,7 +49,10 @@ int internal_memcmp(const void* s1, const void* s2, uptr n) { return 0; } -void *internal_memcpy(void *dest, const void *src, uptr n) { +extern "C" { +SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memcpy(void *dest, + const void *src, + uptr n) { char *d = (char*)dest; const char *s = (const char *)src; for (uptr i = 0; i < n; ++i) @@ -54,7 +60,8 @@ void *internal_memcpy(void *dest, const void *src, uptr n) { return dest; } -void *internal_memmove(void *dest, const void *src, uptr n) { +SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memmove( + void *dest, const void *src, uptr n) { char *d = (char*)dest; const char *s = (const char *)src; sptr i, signed_n = (sptr)n; @@ -72,7 +79,8 @@ void *internal_memmove(void *dest, const void *src, uptr n) { return dest; } -void *internal_memset(void* s, int c, uptr n) { +SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memset(void *s, int c, + uptr n) { // Optimize for the most performance-critical case: if ((reinterpret_cast(s) % 16) == 0 && (n % 16) == 0) { u64 *p = reinterpret_cast(s); @@ -95,6 +103,7 @@ void *internal_memset(void* s, int c, uptr n) { } return s; } +} // extern "C" uptr internal_strcspn(const char *s, const char *reject) { uptr i; diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.h index 39a212665d0ae..e881db2079086 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_libc.h @@ -24,15 +24,33 @@ namespace __sanitizer { // internal_X() is a custom implementation of X() for use in RTL. +extern "C" { +// These are used as builtin replacements; see sanitizer_redefine_builtins.h. +// In normal runtime code, use the __sanitizer::internal_X() aliases instead. +SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memcpy(void *dest, + const void *src, + uptr n); +SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memmove( + void *dest, const void *src, uptr n); +SANITIZER_INTERFACE_ATTRIBUTE void *__sanitizer_internal_memset(void *s, int c, + uptr n); +} // extern "C" + // String functions s64 internal_atoll(const char *nptr); void *internal_memchr(const void *s, int c, uptr n); void *internal_memrchr(const void *s, int c, uptr n); int internal_memcmp(const void* s1, const void* s2, uptr n); -void *internal_memcpy(void *dest, const void *src, uptr n); -void *internal_memmove(void *dest, const void *src, uptr n); +ALWAYS_INLINE void *internal_memcpy(void *dest, const void *src, uptr n) { + return __sanitizer_internal_memcpy(dest, src, n); +} +ALWAYS_INLINE void *internal_memmove(void *dest, const void *src, uptr n) { + return __sanitizer_internal_memmove(dest, src, n); +} // Should not be used in performance-critical places. -void *internal_memset(void *s, int c, uptr n); +ALWAYS_INLINE void *internal_memset(void *s, int c, uptr n) { + return __sanitizer_internal_memset(s, c, n); +} char* internal_strchr(const char *s, int c); char *internal_strchrnul(const char *s, int c); int internal_strcmp(const char *s1, const char *s2); diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp index dada262b02b16..90d5858a72037 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.cpp @@ -168,11 +168,11 @@ const int FUTEX_WAKE_PRIVATE = FUTEX_WAKE | FUTEX_PRIVATE_FLAG; namespace __sanitizer { -void SetSigProcMask(__sanitizer_sigset_t *set, __sanitizer_sigset_t *old) { - CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, set, old)); +void SetSigProcMask(__sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset) { + CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK, set, oldset)); } -ScopedBlockSignals::ScopedBlockSignals(__sanitizer_sigset_t *copy) { +void BlockSignals(__sanitizer_sigset_t *oldset) { __sanitizer_sigset_t set; internal_sigfillset(&set); # if SANITIZER_LINUX && !SANITIZER_ANDROID @@ -187,7 +187,11 @@ ScopedBlockSignals::ScopedBlockSignals(__sanitizer_sigset_t *copy) { // hang. internal_sigdelset(&set, 31); # endif - SetSigProcMask(&set, &saved_); + SetSigProcMask(&set, oldset); +} + +ScopedBlockSignals::ScopedBlockSignals(__sanitizer_sigset_t *copy) { + BlockSignals(&saved_); if (copy) internal_memcpy(copy, &saved_, sizeof(saved_)); } diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h index 87e4959db6fba..d5655eab12a68 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux.h @@ -51,6 +51,7 @@ uptr internal_sigprocmask(int how, __sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset); void SetSigProcMask(__sanitizer_sigset_t *set, __sanitizer_sigset_t *oldset); +void BlockSignals(__sanitizer_sigset_t *oldset = nullptr); struct ScopedBlockSignals { explicit ScopedBlockSignals(__sanitizer_sigset_t *copy); ~ScopedBlockSignals(); @@ -152,6 +153,9 @@ inline void ReleaseMemoryPagesToOSAndZeroFill(uptr beg, uptr end) { "rdhwr %0,$29\n" \ ".set pop\n" : "=r"(__v)); \ __v; }) +#elif defined (__riscv) +# define __get_tls() \ + ({ void** __v; __asm__("mv %0, tp" : "=r"(__v)); __v; }) #elif defined(__i386__) # define __get_tls() \ ({ void** __v; __asm__("movl %%gs:0, %0" : "=r"(__v)); __v; }) diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp index 8777e4c4556e7..1baab2bef68cf 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp @@ -148,7 +148,7 @@ void GetThreadStackTopAndBottom(bool at_initialization, uptr *stack_top, pthread_attr_t attr; pthread_attr_init(&attr); CHECK_EQ(pthread_getattr_np(pthread_self(), &attr), 0); - my_pthread_attr_getstack(&attr, &stackaddr, &stacksize); + internal_pthread_attr_getstack(&attr, &stackaddr, &stacksize); pthread_attr_destroy(&attr); #endif // SANITIZER_SOLARIS diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp index 23c4c6619de82..24e3d1112520e 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mac.cpp @@ -12,81 +12,81 @@ #include "sanitizer_platform.h" #if SANITIZER_APPLE -#include "sanitizer_mac.h" -#include "interception/interception.h" +# include "interception/interception.h" +# include "sanitizer_mac.h" // Use 64-bit inodes in file operations. ASan does not support OS X 10.5, so // the clients will most certainly use 64-bit ones as well. -#ifndef _DARWIN_USE_64_BIT_INODE -#define _DARWIN_USE_64_BIT_INODE 1 -#endif -#include - -#include "sanitizer_common.h" -#include "sanitizer_file.h" -#include "sanitizer_flags.h" -#include "sanitizer_interface_internal.h" -#include "sanitizer_internal_defs.h" -#include "sanitizer_libc.h" -#include "sanitizer_platform_limits_posix.h" -#include "sanitizer_procmaps.h" -#include "sanitizer_ptrauth.h" - -#if !SANITIZER_IOS -#include // for _NSGetEnviron -#else +# ifndef _DARWIN_USE_64_BIT_INODE +# define _DARWIN_USE_64_BIT_INODE 1 +# endif +# include + +# include "sanitizer_common.h" +# include "sanitizer_file.h" +# include "sanitizer_flags.h" +# include "sanitizer_interface_internal.h" +# include "sanitizer_internal_defs.h" +# include "sanitizer_libc.h" +# include "sanitizer_platform_limits_posix.h" +# include "sanitizer_procmaps.h" +# include "sanitizer_ptrauth.h" + +# if !SANITIZER_IOS +# include // for _NSGetEnviron +# else extern char **environ; -#endif +# endif -#if defined(__has_include) && __has_include() -#define SANITIZER_OS_TRACE 1 -#include -#else -#define SANITIZER_OS_TRACE 0 -#endif +# if defined(__has_include) && __has_include() +# define SANITIZER_OS_TRACE 1 +# include +# else +# define SANITIZER_OS_TRACE 0 +# endif // import new crash reporting api -#if defined(__has_include) && __has_include() -#define HAVE_CRASHREPORTERCLIENT_H 1 -#include -#else -#define HAVE_CRASHREPORTERCLIENT_H 0 -#endif +# if defined(__has_include) && __has_include() +# define HAVE_CRASHREPORTERCLIENT_H 1 +# include +# else +# define HAVE_CRASHREPORTERCLIENT_H 0 +# endif -#if !SANITIZER_IOS -#include // for _NSGetArgv and _NSGetEnviron -#else +# if !SANITIZER_IOS +# include // for _NSGetArgv and _NSGetEnviron +# else extern "C" { - extern char ***_NSGetArgv(void); +extern char ***_NSGetArgv(void); } -#endif +# endif -#include -#include // for dladdr() -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +# include +# include // for dladdr() +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include +# include // From , but we don't have that file on iOS. extern "C" { @@ -989,7 +989,7 @@ static void VerifyInterceptorsWorking() { // "wrap_puts" within our own dylib. Dl_info info_puts, info_runtime; RAW_CHECK(dladdr(dlsym(RTLD_DEFAULT, "puts"), &info_puts)); - RAW_CHECK(dladdr((void *)__sanitizer_report_error_summary, &info_runtime)); + RAW_CHECK(dladdr((void *)&VerifyInterceptorsWorking, &info_runtime)); if (internal_strcmp(info_puts.dli_fname, info_runtime.dli_fname) != 0) { Report( "ERROR: Interceptors are not working. This may be because %s is " @@ -1039,7 +1039,7 @@ static void StripEnv() { return; Dl_info info; - RAW_CHECK(dladdr((void *)__sanitizer_report_error_summary, &info)); + RAW_CHECK(dladdr((void *)&StripEnv, &info)); const char *dylib_name = StripModuleName(info.dli_fname); bool lib_is_in_env = internal_strstr(dyld_insert_libraries, dylib_name); if (!lib_is_in_env) diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mallinfo.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mallinfo.h new file mode 100644 index 0000000000000..4e58c02df8351 --- /dev/null +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_mallinfo.h @@ -0,0 +1,38 @@ +//===-- sanitizer_mallinfo.h ----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is a part of Sanitizer common code. +// +// Definition for mallinfo on different platforms. +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_MALLINFO_H +#define SANITIZER_MALLINFO_H + +#include "sanitizer_internal_defs.h" +#include "sanitizer_platform.h" + +namespace __sanitizer { + +#if SANITIZER_ANDROID + +struct __sanitizer_struct_mallinfo { + uptr v[10]; +}; + +#elif SANITIZER_LINUX || SANITIZER_APPLE || SANITIZER_FUCHSIA + +struct __sanitizer_struct_mallinfo { + int v[10]; +}; + +#endif + +} // namespace __sanitizer + +#endif // SANITIZER_MALLINFO_H diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h index 36acd5f2d3395..0df653463f436 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_interceptors.h @@ -359,7 +359,8 @@ #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETINHERITSCHED \ (SI_FREEBSD || SI_NETBSD || SI_MAC || SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GETAFFINITY_NP SI_GLIBC -#define SANITIZER_INTERCEPT_PTHREAD_GETAFFINITY_NP SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_PTHREAD_GETAFFINITY_NP \ + (SI_LINUX_NOT_ANDROID || SI_FREEBSD) #define SANITIZER_INTERCEPT_PTHREAD_ATTR_GET_SCHED SI_POSIX #define SANITIZER_INTERCEPT_PTHREAD_MUTEXATTR_GETPSHARED \ (SI_POSIX && !SI_NETBSD) @@ -379,6 +380,8 @@ (SI_LINUX_NOT_ANDROID || SI_SOLARIS) #define SANITIZER_INTERCEPT_PTHREAD_BARRIERATTR_GETPSHARED \ (SI_LINUX_NOT_ANDROID && !SI_NETBSD) +#define SANITIZER_INTERCEPT_TRYJOIN SI_GLIBC +#define SANITIZER_INTERCEPT_TIMEDJOIN SI_GLIBC #define SANITIZER_INTERCEPT_THR_EXIT SI_FREEBSD #define SANITIZER_INTERCEPT_TMPNAM SI_POSIX #define SANITIZER_INTERCEPT_TMPNAM_R (SI_GLIBC || SI_SOLARIS) @@ -504,6 +507,7 @@ #define SANITIZER_INTERCEPT_ALIGNED_ALLOC (!SI_MAC) #define SANITIZER_INTERCEPT_MALLOC_USABLE_SIZE (!SI_MAC && !SI_NETBSD) #define SANITIZER_INTERCEPT_MCHECK_MPROBE SI_LINUX_NOT_ANDROID +#define SANITIZER_INTERCEPT_WCSLEN 1 #define SANITIZER_INTERCEPT_WCSCAT SI_POSIX #define SANITIZER_INTERCEPT_WCSDUP SI_POSIX #define SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION (!SI_WINDOWS && SI_NOT_FUCHSIA) @@ -581,15 +585,14 @@ #define SANITIZER_INTERCEPT_FDEVNAME SI_FREEBSD #define SANITIZER_INTERCEPT_GETUSERSHELL (SI_POSIX && !SI_ANDROID) #define SANITIZER_INTERCEPT_SL_INIT (SI_FREEBSD || SI_NETBSD) -#define SANITIZER_INTERCEPT_CRYPT (SI_POSIX && !SI_ANDROID) -#define SANITIZER_INTERCEPT_CRYPT_R (SI_LINUX && !SI_ANDROID) #define SANITIZER_INTERCEPT_GETRANDOM \ ((SI_LINUX && __GLIBC_PREREQ(2, 25)) || SI_FREEBSD) #define SANITIZER_INTERCEPT___CXA_ATEXIT SI_NETBSD #define SANITIZER_INTERCEPT_ATEXIT SI_NETBSD #define SANITIZER_INTERCEPT_PTHREAD_ATFORK SI_NETBSD -#define SANITIZER_INTERCEPT_GETENTROPY SI_FREEBSD +#define SANITIZER_INTERCEPT_GETENTROPY \ + ((SI_LINUX && __GLIBC_PREREQ(2, 25)) || SI_FREEBSD) #define SANITIZER_INTERCEPT_QSORT \ (SI_POSIX && !SI_IOSSIM && !SI_WATCHOS && !SI_TVOS && !SI_ANDROID) #define SANITIZER_INTERCEPT_QSORT_R SI_GLIBC @@ -604,6 +607,8 @@ #define SANITIZER_INTERCEPT_FLOPEN SI_FREEBSD #define SANITIZER_INTERCEPT_PROCCTL SI_FREEBSD #define SANITIZER_INTERCEPT_HEXDUMP SI_FREEBSD +#define SANITIZER_INTERCEPT_ARGP_PARSE SI_GLIBC +#define SANITIZER_INTERCEPT_CPUSET_GETAFFINITY SI_FREEBSD // This macro gives a way for downstream users to override the above // interceptor macros irrespective of the platform they are on. They have diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cpp index 37e72cd5d45ea..38f968d533b14 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -103,6 +104,7 @@ void *__sanitizer_get_link_map_by_dlopen_handle(void *handle) { return internal_dlinfo(handle, RTLD_DI_LINKMAP, &p) == 0 ? p : nullptr; } +unsigned struct_cpuset_sz = sizeof(cpuset_t); unsigned struct_cap_rights_sz = sizeof(cap_rights_t); unsigned struct_utsname_sz = sizeof(struct utsname); unsigned struct_stat_sz = sizeof(struct stat); @@ -173,6 +175,12 @@ uptr __sanitizer_in_addr_sz(int af) { return 0; } +// For FreeBSD the actual size of a directory entry is not always in d_reclen. +// Use the appropriate macro to get the correct size for all cases (e.g. NFS). +u16 __sanitizer_dirsiz(const __sanitizer_dirent *dp) { + return _GENERIC_DIRSIZ(dp); +} + unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); int glob_nomatch = GLOB_NOMATCH; int glob_altdirfunc = GLOB_ALTDIRFUNC; @@ -558,4 +566,5 @@ COMPILER_CHECK(__sanitizer_XDR_FREE == XDR_FREE); CHECK_TYPE_SIZE(sem_t); COMPILER_CHECK(sizeof(__sanitizer_cap_rights_t) >= sizeof(cap_rights_t)); +COMPILER_CHECK(sizeof(__sanitizer_cpuset_t) >= sizeof(cpuset_t)); #endif // SANITIZER_FREEBSD diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h index daef1177a2dba..b119f059007d8 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_freebsd.h @@ -249,9 +249,15 @@ struct __sanitizer_dirent { unsigned int d_fileno; # endif unsigned short d_reclen; - // more fields that we don't care about + u8 d_type; + u8 d_pad0; + u16 d_namlen; + u16 d_pad1; + char d_name[256]; }; +u16 __sanitizer_dirsiz(const __sanitizer_dirent *dp); + // 'clock_t' is 32 bits wide on x64 FreeBSD typedef int __sanitizer_clock_t; typedef int __sanitizer_clockid_t; @@ -709,6 +715,17 @@ extern unsigned struct_cap_rights_sz; extern unsigned struct_fstab_sz; extern unsigned struct_StringList_sz; + +struct __sanitizer_cpuset { +#if __FreeBSD_version >= 1400090 + long __bits[(1024 + (sizeof(long) * 8) - 1) / (sizeof(long) * 8)]; +#else + long __bits[(256 + (sizeof(long) * 8) - 1) / (sizeof(long) * 8)]; +#endif +}; + +typedef struct __sanitizer_cpuset __sanitizer_cpuset_t; +extern unsigned struct_cpuset_sz; } // namespace __sanitizer # define CHECK_TYPE_SIZE(TYPE) \ diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cpp index 648e502b904a5..c40877ba48d07 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.cpp @@ -2342,8 +2342,6 @@ unsigned IOCTL_TIOCDRAIN = TIOCDRAIN; unsigned IOCTL_TIOCGFLAGS = TIOCGFLAGS; unsigned IOCTL_TIOCSFLAGS = TIOCSFLAGS; unsigned IOCTL_TIOCDCDTIMESTAMP = TIOCDCDTIMESTAMP; -unsigned IOCTL_TIOCRCVFRAME = TIOCRCVFRAME; -unsigned IOCTL_TIOCXMTFRAME = TIOCXMTFRAME; unsigned IOCTL_TIOCPTMGET = TIOCPTMGET; unsigned IOCTL_TIOCGRANTPT = TIOCGRANTPT; unsigned IOCTL_TIOCPTSNAME = TIOCPTSNAME; diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h index dc6eb59b2800e..4c697b4d107dd 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_netbsd.h @@ -2195,8 +2195,6 @@ extern unsigned IOCTL_TIOCDRAIN; extern unsigned IOCTL_TIOCGFLAGS; extern unsigned IOCTL_TIOCSFLAGS; extern unsigned IOCTL_TIOCDCDTIMESTAMP; -extern unsigned IOCTL_TIOCRCVFRAME; -extern unsigned IOCTL_TIOCXMTFRAME; extern unsigned IOCTL_TIOCPTMGET; extern unsigned IOCTL_TIOCGRANTPT; extern unsigned IOCTL_TIOCPTSNAME; diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp index 8db81c51a5a49..72d76bb581531 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.cpp @@ -18,6 +18,7 @@ // depends on _FILE_OFFSET_BITS setting. // To get this "true" dirent definition, we undefine _FILE_OFFSET_BITS below. #undef _FILE_OFFSET_BITS +#undef _TIME_BITS #endif // Must go after undef _FILE_OFFSET_BITS. @@ -176,10 +177,6 @@ typedef struct user_fpregs elf_fpregset_t; # include "sanitizer_platform_interceptors.h" # include "sanitizer_platform_limits_posix.h" -#if SANITIZER_INTERCEPT_CRYPT_R -#include -#endif - namespace __sanitizer { unsigned struct_utsname_sz = sizeof(struct utsname); unsigned struct_stat_sz = sizeof(struct stat); @@ -247,7 +244,23 @@ namespace __sanitizer { unsigned struct_sysinfo_sz = sizeof(struct sysinfo); unsigned __user_cap_header_struct_sz = sizeof(struct __user_cap_header_struct); - unsigned __user_cap_data_struct_sz = sizeof(struct __user_cap_data_struct); + unsigned __user_cap_data_struct_sz(void *hdrp) { + int u32s = 0; + if (hdrp) { + switch (((struct __user_cap_header_struct *)hdrp)->version) { + case _LINUX_CAPABILITY_VERSION_1: + u32s = _LINUX_CAPABILITY_U32S_1; + break; + case _LINUX_CAPABILITY_VERSION_2: + u32s = _LINUX_CAPABILITY_U32S_2; + break; + case _LINUX_CAPABILITY_VERSION_3: + u32s = _LINUX_CAPABILITY_U32S_3; + break; + } + } + return sizeof(struct __user_cap_data_struct) * u32s; + } unsigned struct_new_utsname_sz = sizeof(struct new_utsname); unsigned struct_old_utsname_sz = sizeof(struct old_utsname); unsigned struct_oldold_utsname_sz = sizeof(struct oldold_utsname); @@ -283,10 +296,6 @@ namespace __sanitizer { unsigned struct_statvfs64_sz = sizeof(struct statvfs64); #endif // SANITIZER_GLIBC -#if SANITIZER_INTERCEPT_CRYPT_R - unsigned struct_crypt_data_sz = sizeof(struct crypt_data); -#endif - #if SANITIZER_LINUX && !SANITIZER_ANDROID unsigned struct_timex_sz = sizeof(struct timex); unsigned struct_msqid_ds_sz = sizeof(struct msqid_ds); diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h index 11d39b80717f0..08d1a685208be 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_platform_limits_posix.h @@ -18,6 +18,7 @@ #include "sanitizer_internal_defs.h" #include "sanitizer_platform.h" +#include "sanitizer_mallinfo.h" #if SANITIZER_APPLE #include @@ -139,7 +140,7 @@ struct __sanitizer_perf_event_attr { extern unsigned struct_epoll_event_sz; extern unsigned struct_sysinfo_sz; extern unsigned __user_cap_header_struct_sz; -extern unsigned __user_cap_data_struct_sz; +extern unsigned __user_cap_data_struct_sz(void *hdrp); extern unsigned struct_new_utsname_sz; extern unsigned struct_old_utsname_sz; extern unsigned struct_oldold_utsname_sz; @@ -208,17 +209,7 @@ struct __sanitizer_sem_t { }; #endif // SANITIZER_LINUX -#if SANITIZER_ANDROID -struct __sanitizer_struct_mallinfo { - uptr v[10]; -}; -#endif - #if SANITIZER_LINUX && !SANITIZER_ANDROID -struct __sanitizer_struct_mallinfo { - int v[10]; -}; - extern unsigned struct_ustat_sz; extern unsigned struct_rlimit64_sz; extern unsigned struct_statvfs64_sz; @@ -322,7 +313,6 @@ extern unsigned struct_msqid_ds_sz; extern unsigned struct_mq_attr_sz; extern unsigned struct_timex_sz; extern unsigned struct_statvfs_sz; -extern unsigned struct_crypt_data_sz; #endif // SANITIZER_LINUX && !SANITIZER_ANDROID struct __sanitizer_iovec { @@ -596,8 +586,13 @@ typedef unsigned long __sanitizer_sigset_t; #endif struct __sanitizer_siginfo_pad { +#if SANITIZER_X32 + // x32 siginfo_t is aligned to 8 bytes. + u64 pad[128 / sizeof(u64)]; +#else // Require uptr, because siginfo_t is always pointer-size aligned on Linux. uptr pad[128 / sizeof(uptr)]; +#endif }; #if SANITIZER_LINUX diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp index 00e49ecbf327e..8159bc0b92c10 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp @@ -57,11 +57,9 @@ void *MmapOrDie(uptr size, const char *mem_type, bool raw_report) { void UnmapOrDie(void *addr, uptr size) { if (!addr || !size) return; uptr res = internal_munmap(addr, size); - if (UNLIKELY(internal_iserror(res))) { - Report("ERROR: %s failed to deallocate 0x%zx (%zd) bytes at address %p\n", - SanitizerToolName, size, size, addr); - CHECK("unable to unmap" && 0); - } + int reserrno; + if (UNLIKELY(internal_iserror(res, &reserrno))) + ReportMunmapFailureAndDie(addr, size, reserrno); DecreaseTotalMmap(size); } @@ -160,6 +158,10 @@ bool MprotectReadOnly(uptr addr, uptr size) { #endif } +bool MprotectReadWrite(uptr addr, uptr size) { + return 0 == internal_mprotect((void *)addr, size, PROT_READ | PROT_WRITE); +} + #if !SANITIZER_APPLE void MprotectMallocZones(void *addr, int prot) {} #endif diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h index f91e26e74b87c..c5811dffea94b 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.h @@ -90,7 +90,7 @@ int real_pthread_join(void *th, void **ret); } \ } // namespace __sanitizer -int my_pthread_attr_getstack(void *attr, void **addr, uptr *size); +int internal_pthread_attr_getstack(void *attr, void **addr, uptr *size); // A routine named real_sigaction() must be implemented by each sanitizer in // order for internal_sigaction() to bypass interceptors. @@ -120,6 +120,9 @@ int GetNamedMappingFd(const char *name, uptr size, int *flags); // alive at least as long as the mapping exists. void DecorateMapping(uptr addr, uptr size, const char *name); +# if !SANITIZER_FREEBSD +# define __sanitizer_dirsiz(dp) ((dp)->d_reclen) +# endif } // namespace __sanitizer diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp index b545453641fca..2c34bcbc8bf0e 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cpp @@ -393,7 +393,7 @@ SANITIZER_WEAK_ATTRIBUTE int real_pthread_attr_getstack(void *attr, void **addr, size_t *size); } // extern "C" -int my_pthread_attr_getstack(void *attr, void **addr, uptr *size) { +int internal_pthread_attr_getstack(void *attr, void **addr, uptr *size) { #if !SANITIZER_GO && !SANITIZER_APPLE if (&real_pthread_attr_getstack) return real_pthread_attr_getstack((pthread_attr_t *)attr, addr, @@ -407,7 +407,7 @@ void AdjustStackSize(void *attr_) { pthread_attr_t *attr = (pthread_attr_t *)attr_; uptr stackaddr = 0; uptr stacksize = 0; - my_pthread_attr_getstack(attr, (void**)&stackaddr, &stacksize); + internal_pthread_attr_getstack(attr, (void **)&stackaddr, &stacksize); // GLibC will return (0 - stacksize) as the stack address in the case when // stacksize is set, but stackaddr is not. bool stack_set = (stackaddr != 0) && (stackaddr + stacksize != 0); diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h index 19bad158387c5..bf3c2c28e32e3 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps.h @@ -65,6 +65,8 @@ class MemoryMappedSegment { MemoryMappedSegmentData *data_; }; +struct ImageHeader; + class MemoryMappingLayoutBase { public: virtual bool Next(MemoryMappedSegment *segment) { UNIMPLEMENTED(); } @@ -75,10 +77,22 @@ class MemoryMappingLayoutBase { ~MemoryMappingLayoutBase() {} }; -class MemoryMappingLayout final : public MemoryMappingLayoutBase { +class MemoryMappingLayout : public MemoryMappingLayoutBase { public: explicit MemoryMappingLayout(bool cache_enabled); + +// This destructor cannot be virtual, as it would cause an operator new() linking +// failures in hwasan test cases. However non-virtual destructors emit warnings +// in macOS build, hence disabling those +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wnon-virtual-dtor" +#endif ~MemoryMappingLayout(); +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + virtual bool Next(MemoryMappedSegment *segment) override; virtual bool Error() const override; virtual void Reset() override; @@ -90,10 +104,14 @@ class MemoryMappingLayout final : public MemoryMappingLayoutBase { // Adds all mapped objects into a vector. void DumpListOfModules(InternalMmapVectorNoCtor *modules); + protected: +#if SANITIZER_APPLE + virtual const ImageHeader *CurrentImageHeader(); +#endif + MemoryMappingLayoutData data_; + private: void LoadFromCache(); - - MemoryMappingLayoutData data_; }; // Returns code range for the specified module. diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp index 4b0e678197614..b44e016a0e5bc 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_procmaps_mac.cpp @@ -250,7 +250,9 @@ static bool NextSegmentLoad(MemoryMappedSegment *segment, MemoryMappedSegmentData *seg_data, MemoryMappingLayoutData *layout_data) { const char *lc = layout_data->current_load_cmd_addr; + layout_data->current_load_cmd_addr += ((const load_command *)lc)->cmdsize; + layout_data->current_load_cmd_count--; if (((const load_command *)lc)->cmd == kLCSegment) { const SegmentCommand* sc = (const SegmentCommand *)lc; uptr base_virt_addr, addr_mask; @@ -358,11 +360,16 @@ static bool IsModuleInstrumented(const load_command *first_lc) { return false; } +const ImageHeader *MemoryMappingLayout::CurrentImageHeader() { + const mach_header *hdr = (data_.current_image == kDyldImageIdx) + ? get_dyld_hdr() + : _dyld_get_image_header(data_.current_image); + return (const ImageHeader *)hdr; +} + bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { for (; data_.current_image >= kDyldImageIdx; data_.current_image--) { - const mach_header *hdr = (data_.current_image == kDyldImageIdx) - ? get_dyld_hdr() - : _dyld_get_image_header(data_.current_image); + const mach_header *hdr = (const mach_header *)CurrentImageHeader(); if (!hdr) continue; if (data_.current_load_cmd_count < 0) { // Set up for this image; @@ -392,7 +399,7 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { (const load_command *)data_.current_load_cmd_addr); } - for (; data_.current_load_cmd_count >= 0; data_.current_load_cmd_count--) { + while (data_.current_load_cmd_count > 0) { switch (data_.current_magic) { // data_.current_magic may be only one of MH_MAGIC, MH_MAGIC_64. #ifdef MH_MAGIC_64 @@ -413,6 +420,7 @@ bool MemoryMappingLayout::Next(MemoryMappedSegment *segment) { } // If we get here, no more load_cmd's in this image talk about // segments. Go on to the next image. + data_.current_load_cmd_count = -1; // This will trigger loading next image } return false; } diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h index 4aa6054851666..460d96ea681b4 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_quarantine.h @@ -68,10 +68,6 @@ struct QuarantineBatch { COMPILER_CHECK(sizeof(QuarantineBatch) <= (1 << 13)); // 8Kb. -// The callback interface is: -// void Callback::Recycle(Node *ptr); -// void *cb.Allocate(uptr size); -// void cb.Deallocate(void *ptr); template class Quarantine { public: @@ -94,21 +90,20 @@ class Quarantine { recycle_mutex_.Init(); } - uptr GetSize() const { return atomic_load_relaxed(&max_size_); } - uptr GetCacheSize() const { - return atomic_load_relaxed(&max_cache_size_); - } + uptr GetMaxSize() const { return atomic_load_relaxed(&max_size_); } + uptr GetMaxCacheSize() const { return atomic_load_relaxed(&max_cache_size_); } void Put(Cache *c, Callback cb, Node *ptr, uptr size) { - uptr cache_size = GetCacheSize(); - if (cache_size) { + uptr max_cache_size = GetMaxCacheSize(); + if (max_cache_size && size <= GetMaxSize()) { + cb.PreQuarantine(ptr); c->Enqueue(cb, ptr, size); } else { - // GetCacheSize() == 0 only when GetSize() == 0 (see Init). - cb.Recycle(ptr); + // GetMaxCacheSize() == 0 only when GetMaxSize() == 0 (see Init). + cb.RecyclePassThrough(ptr); } // Check cache size anyway to accommodate for runtime cache_size change. - if (c->Size() > cache_size) + if (c->Size() > max_cache_size) Drain(c, cb); } @@ -117,7 +112,7 @@ class Quarantine { SpinMutexLock l(&cache_mutex_); cache_.Transfer(c); } - if (cache_.Size() > GetSize() && recycle_mutex_.TryLock()) + if (cache_.Size() > GetMaxSize() && recycle_mutex_.TryLock()) Recycle(atomic_load_relaxed(&min_size_), cb); } @@ -133,7 +128,7 @@ class Quarantine { void PrintStats() const { // It assumes that the world is stopped, just as the allocator's PrintStats. Printf("Quarantine limits: global: %zdMb; thread local: %zdKb\n", - GetSize() >> 20, GetCacheSize() >> 10); + GetMaxSize() >> 20, GetMaxCacheSize() >> 10); cache_.PrintStats(); } diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_range.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_range.cpp new file mode 100644 index 0000000000000..68d79f18ac8d3 --- /dev/null +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_range.cpp @@ -0,0 +1,62 @@ +//===-- sanitizer_range.cpp -----------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "sanitizer_range.h" + +#include "sanitizer_common/sanitizer_array_ref.h" + +namespace __sanitizer { + +void Intersect(ArrayRef a, ArrayRef b, + InternalMmapVectorNoCtor &output) { + output.clear(); + + struct Event { + uptr val; + s8 diff1; + s8 diff2; + }; + + InternalMmapVector events; + for (const Range &r : a) { + CHECK_LE(r.begin, r.end); + events.push_back({r.begin, 1, 0}); + events.push_back({r.end, -1, 0}); + } + + for (const Range &r : b) { + CHECK_LE(r.begin, r.end); + events.push_back({r.begin, 0, 1}); + events.push_back({r.end, 0, -1}); + } + + Sort(events.data(), events.size(), + [](const Event &lh, const Event &rh) { return lh.val < rh.val; }); + + uptr start = 0; + sptr state1 = 0; + sptr state2 = 0; + for (const auto &e : events) { + if (e.val != start) { + DCHECK_GE(state1, 0); + DCHECK_GE(state2, 0); + if (state1 && state2) { + if (!output.empty() && start == output.back().end) + output.back().end = e.val; + else + output.push_back({start, e.val}); + } + start = e.val; + } + + state1 += e.diff1; + state2 += e.diff2; + } +} + +} // namespace __sanitizer diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_range.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_range.h new file mode 100644 index 0000000000000..7c593e171ba21 --- /dev/null +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_range.h @@ -0,0 +1,40 @@ +//===-- sanitizer_range.h ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contais Range and related utilities. +// +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_RANGE_H +#define SANITIZER_RANGE_H + +#include "sanitizer_common.h" +#include "sanitizer_common/sanitizer_array_ref.h" + +namespace __sanitizer { + +struct Range { + uptr begin; + uptr end; +}; + +inline bool operator==(const Range &lhs, const Range &rhs) { + return lhs.begin == rhs.begin && lhs.end == rhs.end; +} + +inline bool operator!=(const Range &lhs, const Range &rhs) { + return !(lhs == rhs); +} + +// Calculates intersection of two sets of regions in O(N log N) time. +void Intersect(ArrayRef a, ArrayRef b, + InternalMmapVectorNoCtor &output); + +} // namespace __sanitizer + +#endif // SANITIZER_RANGE_H diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h new file mode 100644 index 0000000000000..6649ff5844f5c --- /dev/null +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h @@ -0,0 +1,52 @@ +//===-- sanitizer_redefine_builtins.h ---------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Redefine builtin functions to use internal versions. This is needed where +// compiler optimizations end up producing unwanted libcalls! +// +//===----------------------------------------------------------------------===// +#ifndef SANITIZER_COMMON_NO_REDEFINE_BUILTINS +#ifndef SANITIZER_REDEFINE_BUILTINS_H +#define SANITIZER_REDEFINE_BUILTINS_H + +// The asm hack only works with GCC and Clang. +#if !defined(_WIN32) + +asm("memcpy = __sanitizer_internal_memcpy"); +asm("memmove = __sanitizer_internal_memmove"); +asm("memset = __sanitizer_internal_memset"); + +// The builtins should not be redefined in source files that make use of C++ +// standard libraries, in particular where C++STL headers with inline functions +// are used. The redefinition in such cases would lead to ODR violations. +// +// Try to break the build in common cases where builtins shouldn't be redefined. +namespace std { +class Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file { + Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file( + const Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file&) = delete; + Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file& operator=( + const Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file&) = delete; +}; +using array = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using atomic = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using function = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using map = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using set = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using shared_ptr = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using string = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using unique_ptr = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using unordered_map = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using unordered_set = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +using vector = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; +} // namespace std + +#endif // !_WIN32 + +#endif // SANITIZER_REDEFINE_BUILTINS_H +#endif // SANITIZER_COMMON_NO_REDEFINE_BUILTINS diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_signal_interceptors.inc b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_signal_interceptors.inc index 475e577d9982e..94e4e2954a3b9 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_signal_interceptors.inc +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_signal_interceptors.inc @@ -43,6 +43,7 @@ using namespace __sanitizer; #if SANITIZER_INTERCEPT_BSD_SIGNAL INTERCEPTOR(uptr, bsd_signal, int signum, uptr handler) { + SIGNAL_INTERCEPTOR_ENTER(); if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return 0; SIGNAL_INTERCEPTOR_SIGNAL_IMPL(bsd_signal, signum, handler); } @@ -53,6 +54,7 @@ INTERCEPTOR(uptr, bsd_signal, int signum, uptr handler) { #if SANITIZER_INTERCEPT_SIGNAL_AND_SIGACTION INTERCEPTOR(uptr, signal, int signum, uptr handler) { + SIGNAL_INTERCEPTOR_ENTER(); if (GetHandleSignalMode(signum) == kHandleSignalExclusive) return (uptr) nullptr; SIGNAL_INTERCEPTOR_SIGNAL_IMPL(signal, signum, handler); @@ -61,6 +63,7 @@ INTERCEPTOR(uptr, signal, int signum, uptr handler) { INTERCEPTOR(int, sigaction_symname, int signum, const __sanitizer_sigaction *act, __sanitizer_sigaction *oldact) { + SIGNAL_INTERCEPTOR_ENTER(); if (GetHandleSignalMode(signum) == kHandleSignalExclusive) { if (!oldact) return 0; act = nullptr; diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h index 23acbff79c664..75ff41ed52c24 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace.h @@ -94,10 +94,10 @@ uptr StackTrace::GetPreviousInstructionPc(uptr pc) { #elif defined(__sparc__) || defined(__mips__) return pc - 8; #elif SANITIZER_RISCV64 - // RV-64 has variable instruciton length... + // RV-64 has variable instruction length... // C extentions gives us 2-byte instructoins // RV-64 has 4-byte instructions - // + RISCV architecture allows instructions up to 8 bytes + // + RISC-V architecture allows instructions up to 8 bytes // It seems difficult to figure out the exact instruction length - // pc - 2 seems like a safe option for the purposes of stack tracing return pc - 2; diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp index 2d0eccc1602ab..45c480d225c7f 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.cpp @@ -11,25 +11,47 @@ //===----------------------------------------------------------------------===// #include "sanitizer_stacktrace_printer.h" + #include "sanitizer_file.h" +#include "sanitizer_flags.h" #include "sanitizer_fuchsia.h" namespace __sanitizer { -// sanitizer_symbolizer_markup.cpp implements these differently. -#if !SANITIZER_SYMBOLIZER_MARKUP - -static const char *StripFunctionName(const char *function, const char *prefix) { - if (!function) return nullptr; - if (!prefix) return function; - uptr prefix_len = internal_strlen(prefix); - if (0 == internal_strncmp(function, prefix, prefix_len)) - return function + prefix_len; +const char *StripFunctionName(const char *function) { + if (!common_flags()->demangle) + return function; + if (!function) + return nullptr; + auto try_strip = [function](const char *prefix) -> const char * { + const uptr prefix_len = internal_strlen(prefix); + if (!internal_strncmp(function, prefix, prefix_len)) + return function + prefix_len; + return nullptr; + }; + if (SANITIZER_APPLE) { + if (const char *s = try_strip("wrap_")) + return s; + } else if (SANITIZER_WINDOWS) { + if (const char *s = try_strip("__asan_wrap_")) + return s; + } else { + if (const char *s = try_strip("___interceptor_")) + return s; + if (const char *s = try_strip("__interceptor_")) + return s; + } return function; } +// sanitizer_symbolizer_markup.cpp implements these differently. +#if !SANITIZER_SYMBOLIZER_MARKUP + static const char *DemangleFunctionName(const char *function) { - if (!function) return nullptr; + if (!common_flags()->demangle) + return function; + if (!function) + return nullptr; // NetBSD uses indirection for old threading functions for historical reasons // The mangled names are internal implementation detail and should not be @@ -121,7 +143,7 @@ static const char kDefaultFormat[] = " #%n %p %F %L"; void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, uptr address, const AddressInfo *info, bool vs_style, - const char *strip_path_prefix, const char *strip_func_prefix) { + const char *strip_path_prefix) { // info will be null in the case where symbolization is not needed for the // given format. This ensures that the code below will get a hard failure // rather than print incorrect information in case RenderNeedsSymbolization @@ -157,8 +179,8 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/false, buffer); break; case 'f': - buffer->append("%s", DemangleFunctionName(StripFunctionName( - info->function, strip_func_prefix))); + buffer->append("%s", + DemangleFunctionName(StripFunctionName(info->function))); break; case 'q': buffer->append("0x%zx", info->function_offset != AddressInfo::kUnknown @@ -178,8 +200,8 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, case 'F': // Function name and offset, if file is unknown. if (info->function) { - buffer->append("in %s", DemangleFunctionName(StripFunctionName( - info->function, strip_func_prefix))); + buffer->append("in %s", + DemangleFunctionName(StripFunctionName(info->function))); if (!info->file && info->function_offset != AddressInfo::kUnknown) buffer->append("+0x%zx", info->function_offset); } @@ -198,7 +220,9 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, RenderModuleLocation(buffer, info->module, info->module_offset, info->module_arch, strip_path_prefix); +#if !SANITIZER_APPLE MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/true, buffer); +#endif } else { buffer->append("()"); } @@ -211,7 +235,9 @@ void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, // Always strip the module name for %M. RenderModuleLocation(buffer, StripModuleName(info->module), info->module_offset, info->module_arch, ""); +#if !SANITIZER_APPLE MaybeBuildIdToBuffer(*info, /*PrefixSpace=*/true, buffer); +#endif } else { buffer->append("(%p)", (void *)address); } diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h index 96119b2ee9e9f..bf2755a2e8f45 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_printer.h @@ -17,6 +17,9 @@ namespace __sanitizer { +// Strip interceptor prefixes from function name. +const char *StripFunctionName(const char *function); + // Render the contents of "info" structure, which represents the contents of // stack frame "frame_no" and appends it to the "buffer". "format" is a // string with placeholders, which is copied to the output with @@ -26,8 +29,7 @@ namespace __sanitizer { // will be turned into // " frame 10: function foo::bar() at my/file.cc:10" // You may additionally pass "strip_path_prefix" to strip prefixes of paths to -// source files and modules, and "strip_func_prefix" to strip prefixes of -// function names. +// source files and modules. // Here's the full list of available placeholders: // %% - represents a '%' character; // %n - frame number (copy of frame_no); @@ -48,8 +50,7 @@ namespace __sanitizer { // %M - prints module basename and offset, if it is known, or PC. void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, uptr address, const AddressInfo *info, bool vs_style, - const char *strip_path_prefix = "", - const char *strip_func_prefix = ""); + const char *strip_path_prefix = ""); bool RenderNeedsSymbolization(const char *format); diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_sparc.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_sparc.cpp index 1e635a66978f3..a2000798a3907 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_sparc.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stacktrace_sparc.cpp @@ -30,13 +30,7 @@ void BufferedStackTrace::UnwindFast(uptr pc, uptr bp, uptr stack_top, // TODO(yln): add arg sanity check for stack_top/stack_bottom CHECK_GE(max_depth, 2); const uptr kPageSize = GetPageSizeCached(); -#if defined(__GNUC__) - // __builtin_return_address returns the address of the call instruction - // on the SPARC and not the return address, so we need to compensate. - trace_buffer[0] = GetNextInstructionPc(pc); -#else trace_buffer[0] = pc; -#endif size = 1; if (stack_top < 4096) return; // Sanity check for stack top. // Flush register windows to memory diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cpp index 3ebeac52280a3..813616467656b 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_stoptheworld_mac.cpp @@ -154,12 +154,10 @@ PtraceRegistersStatus SuspendedThreadsListMac::GetRegistersAndSP( ®_count); if (err != KERN_SUCCESS) { VReport(1, "Error - unable to get registers for a thread\n"); - // KERN_INVALID_ARGUMENT indicates that either the flavor is invalid, - // or the thread does not exist. The other possible error case, // MIG_ARRAY_TOO_LARGE, means that the state is too large, but it's // still safe to proceed. - return err == KERN_INVALID_ARGUMENT ? REGISTERS_UNAVAILABLE_FATAL - : REGISTERS_UNAVAILABLE; + return err == MIG_ARRAY_TOO_LARGE ? REGISTERS_UNAVAILABLE + : REGISTERS_UNAVAILABLE_FATAL; } buffer->resize(RoundUpTo(sizeof(regs), sizeof(uptr)) / sizeof(uptr)); diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cpp index a674034b8e291..f3818526baab1 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_suppressions.cpp @@ -86,6 +86,7 @@ void SuppressionContext::ParseFromFile(const char *filename) { } Parse(file_contents); + UnmapOrDie(file_contents, contents_size); } bool SuppressionContext::Match(const char *str, const char *type, diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h index 6dcd8a2c58a2e..2e6f2d343fade 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_internal.h @@ -13,8 +13,8 @@ #ifndef SANITIZER_SYMBOLIZER_INTERNAL_H #define SANITIZER_SYMBOLIZER_INTERNAL_H -#include "sanitizer_symbolizer.h" #include "sanitizer_file.h" +#include "sanitizer_symbolizer.h" #include "sanitizer_vector.h" namespace __sanitizer { diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp index 27ed222745ec6..cc02c77bccdc9 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_libbacktrace.cpp @@ -11,11 +11,11 @@ // Libbacktrace implementation of symbolizer parts. //===----------------------------------------------------------------------===// -#include "sanitizer_platform.h" +#include "sanitizer_symbolizer_libbacktrace.h" #include "sanitizer_internal_defs.h" +#include "sanitizer_platform.h" #include "sanitizer_symbolizer.h" -#include "sanitizer_symbolizer_libbacktrace.h" #if SANITIZER_LIBBACKTRACE # include "backtrace-supported.h" diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp index f4f2a036a1e71..a9c958b2d1001 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_mac.cpp @@ -14,16 +14,16 @@ #include "sanitizer_platform.h" #if SANITIZER_APPLE -#include "sanitizer_allocator_internal.h" -#include "sanitizer_mac.h" -#include "sanitizer_symbolizer_mac.h" +# include +# include +# include +# include +# include +# include -#include -#include -#include -#include -#include -#include +# include "sanitizer_allocator_internal.h" +# include "sanitizer_mac.h" +# include "sanitizer_symbolizer_mac.h" namespace __sanitizer { @@ -163,7 +163,7 @@ bool AtosSymbolizer::SymbolizePC(uptr addr, SymbolizedStack *stack) { uptr start_address = AddressInfo::kUnknown; if (!ParseCommandOutput(buf, addr, &stack->info.function, &stack->info.module, &stack->info.file, &line, &start_address)) { - process_ = nullptr; + Report("WARNING: atos failed to symbolize address \"0x%zx\"\n", addr); return false; } stack->info.line = (int)line; diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_markup.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_markup.cpp index 1ec0c5cad7a20..c8c10de10d03a 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_markup.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_markup.cpp @@ -91,7 +91,7 @@ bool RenderNeedsSymbolization(const char *format) { return false; } // We don't support the stack_trace_format flag at all. void RenderFrame(InternalScopedString *buffer, const char *format, int frame_no, uptr address, const AddressInfo *info, bool vs_style, - const char *strip_path_prefix, const char *strip_func_prefix) { + const char *strip_path_prefix) { CHECK(!RenderNeedsSymbolization(format)); buffer->append(kFormatFrame, frame_no, address); } diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp index 7a74f925a17dd..fe532cf09c0fb 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_posix_libcdep.cpp @@ -13,25 +13,25 @@ #include "sanitizer_platform.h" #if SANITIZER_POSIX && !SANITIZER_EMSCRIPTEN -#include "sanitizer_allocator_internal.h" -#include "sanitizer_common.h" -#include "sanitizer_file.h" -#include "sanitizer_flags.h" -#include "sanitizer_internal_defs.h" -#include "sanitizer_linux.h" -#include "sanitizer_placement_new.h" -#include "sanitizer_posix.h" -#include "sanitizer_procmaps.h" -#include "sanitizer_symbolizer_internal.h" -#include "sanitizer_symbolizer_libbacktrace.h" -#include "sanitizer_symbolizer_mac.h" - -#include // for dlsym() -#include -#include -#include -#include -#include +# include // for dlsym() +# include +# include +# include +# include +# include + +# include "sanitizer_allocator_internal.h" +# include "sanitizer_common.h" +# include "sanitizer_file.h" +# include "sanitizer_flags.h" +# include "sanitizer_internal_defs.h" +# include "sanitizer_linux.h" +# include "sanitizer_placement_new.h" +# include "sanitizer_posix.h" +# include "sanitizer_procmaps.h" +# include "sanitizer_symbolizer_internal.h" +# include "sanitizer_symbolizer_libbacktrace.h" +# include "sanitizer_symbolizer_mac.h" // C++ demangling function, as required by Itanium C++ ABI. This is weak, // because we do not require a C++ ABI library to be linked to a program diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp index f23fd0ada8b25..4e43baac3c5e9 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_report.cpp @@ -117,8 +117,7 @@ void ReportMmapWriteExec(int prot, int flags) { stack->Reset(); uptr top = 0; uptr bottom = 0; - GET_CALLER_PC_BP_SP; - (void)sp; + GET_CALLER_PC_BP; bool fast = common_flags()->fast_unwind_on_fatal; if (StackTrace::WillUseFastUnwind(fast)) { GetThreadStackTopAndBottom(false, &top, &bottom); diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp index ac2afe42e2692..ae2d3be19ef38 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_symbolizer_win.cpp @@ -14,8 +14,8 @@ #include "sanitizer_platform.h" #if SANITIZER_WINDOWS -#include "sanitizer_dbghelp.h" -#include "sanitizer_symbolizer_internal.h" +# include "sanitizer_dbghelp.h" +# include "sanitizer_symbolizer_internal.h" namespace __sanitizer { @@ -292,15 +292,15 @@ static void ChooseSymbolizerTools(IntrusiveList *list, const char *path = user_path ? user_path : FindPathToBinary("llvm-symbolizer.exe"); if (path) { - VReport(2, "Using llvm-symbolizer at %spath: %s\n", - user_path ? "user-specified " : "", path); - list->push_back(new(*allocator) LLVMSymbolizer(path, allocator)); - } else { if (user_path && user_path[0] == '\0') { VReport(2, "External symbolizer is explicitly disabled.\n"); } else { - VReport(2, "External symbolizer is not present.\n"); + VReport(2, "Using llvm-symbolizer at %spath: %s\n", + user_path ? "user-specified " : "", path); + list->push_back(new (*allocator) LLVMSymbolizer(path, allocator)); } + } else { + VReport(2, "External symbolizer is not present.\n"); } // Add the dbghelp based symbolizer. diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_arg_retval.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_arg_retval.cpp new file mode 100644 index 0000000000000..bddb285214085 --- /dev/null +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_arg_retval.cpp @@ -0,0 +1,94 @@ +//===-- sanitizer_thread_arg_retval.cpp -------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is shared between sanitizer tools. +// +// Tracks thread arguments and return value for leak checking. +//===----------------------------------------------------------------------===// + +#include "sanitizer_thread_arg_retval.h" + +#include "sanitizer_placement_new.h" + +namespace __sanitizer { + +void ThreadArgRetval::CreateLocked(uptr thread, bool detached, + const Args& args) { + CheckLocked(); + Data& t = data_[thread]; + t = {}; + t.gen = gen_++; + t.detached = detached; + t.args = args; +} + +ThreadArgRetval::Args ThreadArgRetval::GetArgs(uptr thread) const { + __sanitizer::Lock lock(&mtx_); + auto t = data_.find(thread); + CHECK(t); + if (t->second.done) + return {}; + return t->second.args; +} + +void ThreadArgRetval::Finish(uptr thread, void* retval) { + __sanitizer::Lock lock(&mtx_); + auto t = data_.find(thread); + if (!t) + return; + if (t->second.detached) { + // Retval of detached thread connot be retrieved. + data_.erase(t); + return; + } + t->second.done = true; + t->second.args.arg_retval = retval; +} + +u32 ThreadArgRetval::BeforeJoin(uptr thread) const { + __sanitizer::Lock lock(&mtx_); + auto t = data_.find(thread); + CHECK(t); + CHECK(!t->second.detached); + return t->second.gen; +} + +void ThreadArgRetval::AfterJoin(uptr thread, u32 gen) { + __sanitizer::Lock lock(&mtx_); + auto t = data_.find(thread); + if (!t || gen != t->second.gen) { + // Thread was reused and erased by any other event. + return; + } + CHECK(!t->second.detached); + data_.erase(t); +} + +void ThreadArgRetval::DetachLocked(uptr thread) { + CheckLocked(); + auto t = data_.find(thread); + CHECK(t); + CHECK(!t->second.detached); + if (t->second.done) { + // We can't retrive retval after detached thread finished. + data_.erase(t); + return; + } + t->second.detached = true; +} + +void ThreadArgRetval::GetAllPtrsLocked(InternalMmapVector* ptrs) { + CheckLocked(); + CHECK(ptrs); + data_.forEach([&](DenseMap::value_type& kv) -> bool { + ptrs->push_back((uptr)kv.second.args.arg_retval); + return true; + }); +} + +} // namespace __sanitizer diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_arg_retval.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_arg_retval.h new file mode 100644 index 0000000000000..c77021beb67d1 --- /dev/null +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_arg_retval.h @@ -0,0 +1,116 @@ +//===-- sanitizer_thread_arg_retval.h ---------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file is shared between sanitizer tools. +// +// Tracks thread arguments and return value for leak checking. +//===----------------------------------------------------------------------===// + +#ifndef SANITIZER_THREAD_ARG_RETVAL_H +#define SANITIZER_THREAD_ARG_RETVAL_H + +#include "sanitizer_common.h" +#include "sanitizer_dense_map.h" +#include "sanitizer_list.h" +#include "sanitizer_mutex.h" + +namespace __sanitizer { + +// Primary goal of the class is to keep alive arg and retval pointer for leak +// checking. However it can be used to pass those pointer into wrappers used by +// interceptors. The difference from ThreadRegistry/ThreadList is that this +// class keeps data up to the detach or join, as exited thread still can be +// joined to retrive retval. ThreadRegistry/ThreadList can discard exited +// threads immediately. +class SANITIZER_MUTEX ThreadArgRetval { + public: + struct Args { + void* (*routine)(void*); + void* arg_retval; // Either arg or retval. + }; + void Lock() SANITIZER_ACQUIRE() { mtx_.Lock(); } + void CheckLocked() const SANITIZER_CHECK_LOCKED() { mtx_.CheckLocked(); } + void Unlock() SANITIZER_RELEASE() { mtx_.Unlock(); } + + // Wraps pthread_create or similar. We need to keep object locked, to + // prevent child thread from proceeding without thread handle. + template + void Create(bool detached, const Args& args, const CreateFn& fn) { + // No need to track detached threads with no args, but we will to do as it's + // not expensive and less edge-cases. + __sanitizer::Lock lock(&mtx_); + if (uptr thread = fn()) + CreateLocked(thread, detached, args); + } + + // Returns thread arg and routine. + Args GetArgs(uptr thread) const; + + // Mark thread as done and stores retval or remove if detached. Should be + // called by the thread. + void Finish(uptr thread, void* retval); + + // Mark thread as detached or remove if done. + template + void Detach(uptr thread, const DetachFn& fn) { + // Lock to prevent re-use of the thread between fn() and DetachLocked() + // calls. + __sanitizer::Lock lock(&mtx_); + if (fn()) + DetachLocked(thread); + } + + // Joins the thread. + template + void Join(uptr thread, const JoinFn& fn) { + // Remember internal id of the thread to prevent re-use of the thread + // between fn() and AfterJoin() calls. Locking JoinFn, like in + // Detach(), implementation can cause deadlock. + auto gen = BeforeJoin(thread); + if (fn()) + AfterJoin(thread, gen); + } + + // Returns all arg and retval which are considered alive. + void GetAllPtrsLocked(InternalMmapVector* ptrs); + + uptr size() const { + __sanitizer::Lock lock(&mtx_); + return data_.size(); + } + + // FIXME: Add fork support. Expected users of the class are sloppy with forks + // anyway. We likely should lock/unlock the object to avoid deadlocks, and + // erase all but the current threads, so we can detect leaked arg or retval in + // child process. + + // FIXME: Add cancelation support. Now if a thread was canceled, the class + // will keep pointers alive forever, missing leaks caused by cancelation. + + private: + struct Data { + Args args; + u32 gen; // Avoid collision if thread id re-used. + bool detached; + bool done; + }; + + void CreateLocked(uptr thread, bool detached, const Args& args); + u32 BeforeJoin(uptr thread) const; + void AfterJoin(uptr thread, u32 gen); + void DetachLocked(uptr thread); + + mutable Mutex mtx_; + + DenseMap data_; + u32 gen_ = 0; +}; + +} // namespace __sanitizer + +#endif // SANITIZER_THREAD_ARG_RETVAL_H diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp index 278f6defca95f..741e0731c4155 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_thread_registry.cpp @@ -335,7 +335,7 @@ void ThreadRegistry::QuarantinePush(ThreadContextBase *tctx) { ThreadContextBase *ThreadRegistry::QuarantinePop() { if (invalid_threads_.size() == 0) - return 0; + return nullptr; ThreadContextBase *tctx = invalid_threads_.front(); invalid_threads_.pop_front(); return tctx; diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp index b13e2dc9e3327..252979f1c2baa 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.cpp @@ -12,6 +12,7 @@ #include "sanitizer_tls_get_addr.h" +#include "sanitizer_allocator_interface.h" #include "sanitizer_atomic.h" #include "sanitizer_flags.h" #include "sanitizer_platform_interceptors.h" @@ -26,13 +27,6 @@ struct TlsGetAddrParam { uptr offset; }; -// Glibc starting from 2.19 allocates tls using __signal_safe_memalign, -// which has such header. -struct Glibc_2_19_tls_header { - uptr size; - uptr start; -}; - // This must be static TLS __attribute__((tls_model("initial-exec"))) static __thread DTLS dtls; @@ -108,6 +102,14 @@ static const uptr kDtvOffset = 0x800; static const uptr kDtvOffset = 0; #endif +extern "C" { +SANITIZER_WEAK_ATTRIBUTE +uptr __sanitizer_get_allocated_size(const void *p); + +SANITIZER_WEAK_ATTRIBUTE +const void *__sanitizer_get_allocated_begin(const void *p); +} + DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res, uptr static_tls_begin, uptr static_tls_end) { if (!common_flags()->intercept_tls_get_addr) return 0; @@ -125,19 +127,18 @@ DTLS::DTV *DTLS_on_tls_get_addr(void *arg_void, void *res, atomic_load(&number_of_live_dtls, memory_order_relaxed)); if (dtls.last_memalign_ptr == tls_beg) { tls_size = dtls.last_memalign_size; - VReport(2, "__tls_get_addr: glibc <=2.18 suspected; tls={0x%zx,0x%zx}\n", + VReport(2, "__tls_get_addr: glibc <=2.24 suspected; tls={0x%zx,0x%zx}\n", tls_beg, tls_size); } else if (tls_beg >= static_tls_begin && tls_beg < static_tls_end) { // This is the static TLS block which was initialized / unpoisoned at thread // creation. VReport(2, "__tls_get_addr: static tls: 0x%zx\n", tls_beg); tls_size = 0; - } else if ((tls_beg % 4096) == sizeof(Glibc_2_19_tls_header)) { - // We may want to check gnu_get_libc_version(). - Glibc_2_19_tls_header *header = (Glibc_2_19_tls_header *)tls_beg - 1; - tls_size = header->size; - tls_beg = header->start; - VReport(2, "__tls_get_addr: glibc >=2.19 suspected; tls={0x%zx 0x%zx}\n", + } else if (const void *start = + __sanitizer_get_allocated_begin((void *)tls_beg)) { + tls_beg = (uptr)start; + tls_size = __sanitizer_get_allocated_size(start); + VReport(2, "__tls_get_addr: glibc >=2.25 suspected; tls={0x%zx,0x%zx}\n", tls_beg, tls_size); } else { VReport(2, "__tls_get_addr: Can't guess glibc version\n"); diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h index a599c0bbc75cc..0ddab61deb102 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_tls_get_addr.h @@ -12,16 +12,24 @@ // the lack of interface that would tell us about the Dynamic TLS (DTLS). // https://sourceware.org/bugzilla/show_bug.cgi?id=16291 // -// The matters get worse because the glibc implementation changed between -// 2.18 and 2.19: -// https://groups.google.com/forum/#!topic/address-sanitizer/BfwYD8HMxTM -// -// Before 2.19, every DTLS chunk is allocated with __libc_memalign, +// Before 2.25: every DTLS chunk is allocated with __libc_memalign, // which we intercept and thus know where is the DTLS. -// Since 2.19, DTLS chunks are allocated with __signal_safe_memalign, -// which is an internal function that wraps a mmap call, neither of which -// we can intercept. Luckily, __signal_safe_memalign has a simple parseable -// header which we can use. +// +// Since 2.25: DTLS chunks are allocated with malloc. We could co-opt +// the malloc interceptor to keep track of the last allocation, similar +// to how we handle __libc_memalign; however, this adds some overhead +// (since malloc, unlike __libc_memalign, is commonly called), and +// requires care to avoid false negatives for LeakSanitizer. +// Instead, we rely on our internal allocators - which keep track of all +// its allocations - to determine if an address points to a malloc +// allocation. +// +// There exists a since-deprecated version of Google's internal glibc fork +// that used __signal_safe_memalign. DTLS_on_tls_get_addr relied on a +// heuristic check (is the allocation 16 bytes from the start of a page +// boundary?), which was sometimes erroneous: +// https://bugs.chromium.org/p/chromium/issues/detail?id=1275223#c15 +// Since that check has no practical use anymore, we have removed it. // //===----------------------------------------------------------------------===// diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp index 72f025a7d307a..6a8e82e2e213c 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_unwind_linux_libcdep.cpp @@ -139,13 +139,7 @@ void BufferedStackTrace::UnwindSlow(uptr pc, u32 max_depth) { if (to_pop == 0 && size > 1) to_pop = 1; PopStackFrames(to_pop); -#if defined(__GNUC__) && defined(__sparc__) - // __builtin_return_address returns the address of the call instruction - // on the SPARC and not the return address, so we need to compensate. - trace_buffer[0] = GetNextInstructionPc(pc); -#else trace_buffer[0] = pc; -#endif } void BufferedStackTrace::UnwindSlow(uptr pc, void *context, u32 max_depth) { diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp index e0568c9b62d5e..06e496523eeaa 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_win.cpp @@ -362,6 +362,11 @@ bool MprotectReadOnly(uptr addr, uptr size) { return VirtualProtect((LPVOID)addr, size, PAGE_READONLY, &old_protection); } +bool MprotectReadWrite(uptr addr, uptr size) { + DWORD old_protection; + return VirtualProtect((LPVOID)addr, size, PAGE_READWRITE, &old_protection); +} + void ReleaseMemoryPagesToOS(uptr beg, uptr end) { uptr beg_aligned = RoundDownTo(beg, GetPageSizeCached()), end_aligned = RoundDownTo(end, GetPageSizeCached()); @@ -718,13 +723,24 @@ void ListOfModules::fallbackInit() { clear(); } // atexit() as soon as it is ready for use (i.e. after .CRT$XIC initializers). InternalMmapVectorNoCtor atexit_functions; -int Atexit(void (*function)(void)) { +static int queueAtexit(void (*function)(void)) { atexit_functions.push_back(function); return 0; } +// If Atexit() is being called after RunAtexit() has already been run, it needs +// to be able to call atexit() directly. Here we use a function ponter to +// switch out its behaviour. +// An example of where this is needed is the asan_dynamic runtime on MinGW-w64. +// On this environment, __asan_init is called during global constructor phase, +// way after calling the .CRT$XID initializer. +static int (*volatile queueOrCallAtExit)(void (*)(void)) = &queueAtexit; + +int Atexit(void (*function)(void)) { return queueOrCallAtExit(function); } + static int RunAtexit() { TraceLoggingUnregister(g_asan_provider); + queueOrCallAtExit = &atexit; int ret = 0; for (uptr i = 0; i < atexit_functions.size(); ++i) { ret |= atexit(atexit_functions[i]); diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_win_dll_thunk.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_win_dll_thunk.h index 48c73c4c98ad5..639d91a2edaec 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_win_dll_thunk.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_win_dll_thunk.h @@ -84,7 +84,7 @@ extern "C" int __dll_thunk_init(); // which isn't a big deal. #define INTERCEPT_LIBRARY_FUNCTION(name) \ extern "C" void name(); \ - INTERCEPT_OR_DIE(WRAPPER_NAME(name), name) + INTERCEPT_OR_DIE(STRINGIFY(WRAP(name)), name) // Use these macros for functions that could be called before __dll_thunk_init() // is executed and don't lead to errors if defined (free, malloc, etc). diff --git a/system/lib/compiler-rt/lib/ubsan/ubsan_diag.cpp b/system/lib/compiler-rt/lib/ubsan/ubsan_diag.cpp index 8c6651b272e26..673a4d3dbd67c 100644 --- a/system/lib/compiler-rt/lib/ubsan/ubsan_diag.cpp +++ b/system/lib/compiler-rt/lib/ubsan/ubsan_diag.cpp @@ -214,7 +214,12 @@ static void RenderText(InternalScopedString *Buffer, const char *Message, // printf, and stop using snprintf here. char FloatBuffer[32]; #if SANITIZER_WINDOWS - sprintf_s(FloatBuffer, sizeof(FloatBuffer), "%Lg", (long double)A.Float); + // On MSVC platforms, long doubles are equal to regular doubles. + // In MinGW environments on x86, long doubles are 80 bit, but here, + // we're calling an MS CRT provided printf function which considers + // long doubles to be 64 bit. Just cast the float value to a regular + // double to avoid the potential ambiguity in MinGW mode. + sprintf_s(FloatBuffer, sizeof(FloatBuffer), "%g", (double)A.Float); #else snprintf(FloatBuffer, sizeof(FloatBuffer), "%Lg", (long double)A.Float); #endif diff --git a/system/lib/compiler-rt/lib/ubsan/ubsan_handlers.cpp b/system/lib/compiler-rt/lib/ubsan/ubsan_handlers.cpp index a0b869e7ede1b..85342711e318e 100644 --- a/system/lib/compiler-rt/lib/ubsan/ubsan_handlers.cpp +++ b/system/lib/compiler-rt/lib/ubsan/ubsan_handlers.cpp @@ -896,4 +896,39 @@ void __ubsan::__ubsan_handle_cfi_check_fail_abort(CFICheckFailData *Data, Die(); } +static bool handleFunctionTypeMismatch(FunctionTypeMismatchData *Data, + ValueHandle Function, + ReportOptions Opts) { + SourceLocation CallLoc = Data->Loc.acquire(); + ErrorType ET = ErrorType::FunctionTypeMismatch; + if (ignoreReport(CallLoc, Opts, ET)) + return true; + + ScopedReport R(Opts, CallLoc, ET); + + SymbolizedStackHolder FLoc(getSymbolizedLocation(Function)); + const char *FName = FLoc.get()->info.function; + if (!FName) + FName = "(unknown)"; + + Diag(CallLoc, DL_Error, ET, + "call to function %0 through pointer to incorrect function type %1") + << FName << Data->Type; + Diag(FLoc, DL_Note, ET, "%0 defined here") << FName; + return true; +} + +void __ubsan::__ubsan_handle_function_type_mismatch( + FunctionTypeMismatchData *Data, ValueHandle Function) { + GET_REPORT_OPTIONS(false); + handleFunctionTypeMismatch(Data, Function, Opts); +} + +void __ubsan::__ubsan_handle_function_type_mismatch_abort( + FunctionTypeMismatchData *Data, ValueHandle Function) { + GET_REPORT_OPTIONS(true); + if (handleFunctionTypeMismatch(Data, Function, Opts)) + Die(); +} + #endif // CAN_SANITIZE_UB diff --git a/system/lib/compiler-rt/lib/ubsan/ubsan_handlers.h b/system/lib/compiler-rt/lib/ubsan/ubsan_handlers.h index 219fb15de55fe..3bd5046de3d7d 100644 --- a/system/lib/compiler-rt/lib/ubsan/ubsan_handlers.h +++ b/system/lib/compiler-rt/lib/ubsan/ubsan_handlers.h @@ -231,6 +231,17 @@ extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __ubsan_handle_cfi_bad_type( CFICheckFailData *Data, ValueHandle Vtable, bool ValidVtable, ReportOptions Opts); +struct FunctionTypeMismatchData { + SourceLocation Loc; + const TypeDescriptor &Type; +}; + +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void +__ubsan_handle_function_type_mismatch(FunctionTypeMismatchData *Data, + ValueHandle Val); +extern "C" SANITIZER_INTERFACE_ATTRIBUTE void +__ubsan_handle_function_type_mismatch_abort(FunctionTypeMismatchData *Data, + ValueHandle Val); } #endif // UBSAN_HANDLERS_H diff --git a/system/lib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cpp b/system/lib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cpp index 0317a3d1428c8..206a0bb485a9c 100644 --- a/system/lib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cpp +++ b/system/lib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.cpp @@ -156,50 +156,6 @@ void __ubsan_handle_cfi_bad_type(CFICheckFailData *Data, ValueHandle Vtable, Diag(Loc, DL_Note, ET, "check failed in %0, vtable located in %1") << SrcModule << DstModule; } - -static bool handleFunctionTypeMismatch(FunctionTypeMismatchData *Data, - ValueHandle Function, - ValueHandle calleeRTTI, - ValueHandle fnRTTI, ReportOptions Opts) { - if (checkTypeInfoEquality(reinterpret_cast(calleeRTTI), - reinterpret_cast(fnRTTI))) - return false; - - SourceLocation CallLoc = Data->Loc.acquire(); - ErrorType ET = ErrorType::FunctionTypeMismatch; - - if (ignoreReport(CallLoc, Opts, ET)) - return true; - - ScopedReport R(Opts, CallLoc, ET); - - SymbolizedStackHolder FLoc(getSymbolizedLocation(Function)); - const char *FName = FLoc.get()->info.function; - if (!FName) - FName = "(unknown)"; - - Diag(CallLoc, DL_Error, ET, - "call to function %0 through pointer to incorrect function type %1") - << FName << Data->Type; - Diag(FLoc, DL_Note, ET, "%0 defined here") << FName; - return true; -} - -void __ubsan_handle_function_type_mismatch_v1(FunctionTypeMismatchData *Data, - ValueHandle Function, - ValueHandle calleeRTTI, - ValueHandle fnRTTI) { - GET_REPORT_OPTIONS(false); - handleFunctionTypeMismatch(Data, Function, calleeRTTI, fnRTTI, Opts); -} - -void __ubsan_handle_function_type_mismatch_v1_abort( - FunctionTypeMismatchData *Data, ValueHandle Function, - ValueHandle calleeRTTI, ValueHandle fnRTTI) { - GET_REPORT_OPTIONS(true); - if (handleFunctionTypeMismatch(Data, Function, calleeRTTI, fnRTTI, Opts)) - Die(); -} } // namespace __ubsan #endif // CAN_SANITIZE_UB diff --git a/system/lib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.h b/system/lib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.h index fd534c2573f6d..71695cbdc090f 100644 --- a/system/lib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.h +++ b/system/lib/compiler-rt/lib/ubsan/ubsan_handlers_cxx.h @@ -33,22 +33,6 @@ void __ubsan_handle_dynamic_type_cache_miss( extern "C" SANITIZER_INTERFACE_ATTRIBUTE void __ubsan_handle_dynamic_type_cache_miss_abort( DynamicTypeCacheMissData *Data, ValueHandle Pointer, ValueHandle Hash); - -struct FunctionTypeMismatchData { - SourceLocation Loc; - const TypeDescriptor &Type; -}; - -extern "C" SANITIZER_INTERFACE_ATTRIBUTE void -__ubsan_handle_function_type_mismatch_v1(FunctionTypeMismatchData *Data, - ValueHandle Val, - ValueHandle calleeRTTI, - ValueHandle fnRTTI); -extern "C" SANITIZER_INTERFACE_ATTRIBUTE void -__ubsan_handle_function_type_mismatch_v1_abort(FunctionTypeMismatchData *Data, - ValueHandle Val, - ValueHandle calleeRTTI, - ValueHandle fnRTTI); } #endif // UBSAN_HANDLERS_CXX_H diff --git a/system/lib/compiler-rt/lib/ubsan/ubsan_interface.inc b/system/lib/compiler-rt/lib/ubsan/ubsan_interface.inc index 94337d85017b4..cb27feb5d7e99 100644 --- a/system/lib/compiler-rt/lib/ubsan/ubsan_interface.inc +++ b/system/lib/compiler-rt/lib/ubsan/ubsan_interface.inc @@ -21,8 +21,8 @@ INTERFACE_FUNCTION(__ubsan_handle_dynamic_type_cache_miss) INTERFACE_FUNCTION(__ubsan_handle_dynamic_type_cache_miss_abort) INTERFACE_FUNCTION(__ubsan_handle_float_cast_overflow) INTERFACE_FUNCTION(__ubsan_handle_float_cast_overflow_abort) -INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch_v1) -INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch_v1_abort) +INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch) +INTERFACE_FUNCTION(__ubsan_handle_function_type_mismatch_abort) INTERFACE_FUNCTION(__ubsan_handle_implicit_conversion) INTERFACE_FUNCTION(__ubsan_handle_implicit_conversion_abort) INTERFACE_FUNCTION(__ubsan_handle_invalid_builtin) diff --git a/system/lib/compiler-rt/lib/ubsan/ubsan_signals_standalone.cpp b/system/lib/compiler-rt/lib/ubsan/ubsan_signals_standalone.cpp index 8d980e1a26391..fec9ac2e032d2 100644 --- a/system/lib/compiler-rt/lib/ubsan/ubsan_signals_standalone.cpp +++ b/system/lib/compiler-rt/lib/ubsan/ubsan_signals_standalone.cpp @@ -34,7 +34,12 @@ void InitializeDeadlySignals() {} #else +namespace __ubsan { +void InitializeDeadlySignals(); +} // namespace __ubsan + #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name) +#define SIGNAL_INTERCEPTOR_ENTER() __ubsan::InitializeDeadlySignals() #include "sanitizer_common/sanitizer_signal_interceptors.inc" // TODO(yln): Temporary workaround. Will be removed. From 959984daf9ecbeefcfa203c8e333c7d4fca7db14 Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Mon, 13 Nov 2023 21:34:48 -0800 Subject: [PATCH 02/10] Exclude crtbegin.c / crtend.c `crtbegin.c` and `crtend.c` were originally in `compiler-rt/lib/crt`, the directory we don't maintain, and moved to `compiler-rt/lib/builtins` recently. So they were included in the `excludes` list. --- tools/system_libs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/system_libs.py b/tools/system_libs.py index 97c4e1598bdbc..683b99c5ce00d 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -914,7 +914,7 @@ class libcompiler_rt(MTLibrary, SjLjLibrary): src_dir = 'system/lib/compiler-rt/lib/builtins' includes = ['system/lib/libc'] # gcc_personality_v0.c depends on libunwind, which don't include by default. - src_files = glob_in_path(src_dir, '*.c', excludes=['gcc_personality_v0.c', 'truncdfbf2.c', 'truncsfbf2.c']) + src_files = glob_in_path(src_dir, '*.c', excludes=['gcc_personality_v0.c', 'truncdfbf2.c', 'truncsfbf2.c', 'crtbegin.c', 'crtend.c']) src_files += files_in_path( path='system/lib/compiler-rt', filenames=[ From 4ee3a2f27f57d21035e385e41b435c8bee6defcd Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Thu, 2 Nov 2023 13:12:01 -0700 Subject: [PATCH 03/10] Disable asm redefinition of sanitizer symbols This was introduced in https://github.com/llvm/llvm-project/commit/0a71e25e2448ee471b1ebe74e910c5de9b9c82b4, which makes Wasm backend error out. --- .../lib/sanitizer_common/sanitizer_redefine_builtins.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h index 6649ff5844f5c..9e4c2028dc430 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_redefine_builtins.h @@ -15,7 +15,8 @@ #define SANITIZER_REDEFINE_BUILTINS_H // The asm hack only works with GCC and Clang. -#if !defined(_WIN32) +// XXX Emscripten This does not work in Wasm. +#if !defined(_WIN32) && !defined(__wasm__) asm("memcpy = __sanitizer_internal_memcpy"); asm("memmove = __sanitizer_internal_memmove"); @@ -46,7 +47,7 @@ using unordered_set = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; using vector = Define_SANITIZER_COMMON_NO_REDEFINE_BUILTINS_in_cpp_file; } // namespace std -#endif // !_WIN32 +#endif // !_WIN32 && !__wasm__ #endif // SANITIZER_REDEFINE_BUILTINS_H #endif // SANITIZER_COMMON_NO_REDEFINE_BUILTINS From 95b0441abcc7471e137a31622347b947c39c2cb9 Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Thu, 2 Nov 2023 17:45:32 -0700 Subject: [PATCH 04/10] lsan fixes - lsan_common_emscripten.cpp: Function name changes - lsan_common.cpp: The function structure has changed. Previously we had separate `ProcessRootRegion` and `ProcessRootRegions`, and Emscripten modified `ProcessRootRegion`. But in LLVM 17 `ProcessRootRegion` was deleted and merged into `ProcessRootRegions`. This fixes the code according to the new semantics. --- system/lib/compiler-rt/lib/lsan/lsan_common.cpp | 6 +++--- .../lib/compiler-rt/lib/lsan/lsan_common_emscripten.cpp | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common.cpp b/system/lib/compiler-rt/lib/lsan/lsan_common.cpp index 22259e30288fc..d3dfec5ede1b5 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_common.cpp +++ b/system/lib/compiler-rt/lib/lsan/lsan_common.cpp @@ -595,17 +595,17 @@ void ScanRootRegions(Frontier *frontier, static void ProcessRootRegions(Frontier *frontier) { if (!flags()->use_root_regions || !HasRootRegions()) return; + InternalMmapVector mapped_regions; #if SANITIZER_EMSCRIPTEN - ScanRootRegion(frontier, root_region, 0, emscripten_get_heap_size(), true); + mapped_regions.push_back({0, emscripten_get_heap_size()}); #else MemoryMappingLayout proc_maps(/*cache_enabled*/ true); MemoryMappedSegment segment; - InternalMmapVector mapped_regions; while (proc_maps.Next(&segment)) if (segment.IsReadable()) mapped_regions.push_back({segment.start, segment.end}); - ScanRootRegions(frontier, mapped_regions); #endif // SANITIZER_EMSCRIPTEN + ScanRootRegions(frontier, mapped_regions); } static void FloodFillTag(Frontier *frontier, ChunkTag tag) { diff --git a/system/lib/compiler-rt/lib/lsan/lsan_common_emscripten.cpp b/system/lib/compiler-rt/lib/lsan/lsan_common_emscripten.cpp index f2b1301577238..2407842cb5828 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_common_emscripten.cpp +++ b/system/lib/compiler-rt/lib/lsan/lsan_common_emscripten.cpp @@ -105,14 +105,14 @@ void LockStuffAndStopTheWorld(StopTheWorldCallback callback, CheckForLeaksParam *argument) { // Currently, on Emscripten this does nothing and just calls the callback. // This works fine on a single-threaded environment. - LockThreadRegistry(); + LockThreads(); LockAllocator(); StopTheWorld(callback, argument); UnlockAllocator(); - UnlockThreadRegistry(); + UnlockThreads(); } -u32 GetCurrentThread(); +u32 GetCurrentThreadId(); // This is based on ProcessThreads in lsan_common.cc. // We changed this to be a callback that gets called per thread by @@ -142,7 +142,7 @@ static void ProcessThreadsCallback(ThreadContextBase *tctx, void *arg) { // We can't get the SP for other threads to narrow down the range, but we // we can for the current thread. - if (tctx->tid == GetCurrentThread()) { + if (tctx->tid == GetCurrentThreadId()) { uptr sp = (uptr) __builtin_frame_address(0); if (sp < stack_begin || sp >= stack_end) { // SP is outside the recorded stack range (e.g. the thread is running a From 14fad4c382c311e170517f0c6e45b726655b191e Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Fri, 3 Nov 2023 16:47:14 -0700 Subject: [PATCH 05/10] Fix pthread_exit's interceptor's return type These interceptors was added in LLVM 17, but it looks they have a wrong return type. I submitted https://github.com/llvm/llvm-project/pull/71253 to fix that upstream, but in the meantime we should fix this to pass our tests. --- system/lib/compiler-rt/lib/asan/asan_interceptors.cpp | 4 ++-- system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp b/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp index 12f5fe918d95c..bd87c64d78e96 100644 --- a/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp +++ b/system/lib/compiler-rt/lib/asan/asan_interceptors.cpp @@ -286,9 +286,9 @@ INTERCEPTOR(int, pthread_detach, void *thread) { return result; } -INTERCEPTOR(int, pthread_exit, void *retval) { +INTERCEPTOR(void, pthread_exit, void *retval) { asanThreadArgRetval().Finish(GetThreadSelf(), retval); - return REAL(pthread_exit)(retval); + REAL(pthread_exit)(retval); } # if ASAN_INTERCEPT_TRYJOIN diff --git a/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp b/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp index 10494cee230eb..bee31513f1239 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp +++ b/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp @@ -501,9 +501,9 @@ INTERCEPTOR(int, pthread_detach, void *thread) { return result; } -INTERCEPTOR(int, pthread_exit, void *retval) { +INTERCEPTOR(void, pthread_exit, void *retval) { GetThreadArgRetval().Finish(GetThreadSelf(), retval); - return REAL(pthread_exit)(retval); + REAL(pthread_exit)(retval); } # if SANITIZER_INTERCEPT_TRYJOIN From 7fd9a23e7050d6a3d4fd32713f4f7246951aaee0 Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Fri, 3 Nov 2023 13:54:15 -0700 Subject: [PATCH 06/10] Add declaration & aliases for emscripten_builtin_pthread_exit We need these after `pthread_exit` LSan interceptor was added in https://github.com/llvm/llvm-project/commit/da7943b63784050d67874566a498c079c796dc8d. --- system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp | 1 + system/lib/pthread/library_pthread_stub.c | 2 ++ system/lib/pthread/pthread_create.c | 1 + 3 files changed, 4 insertions(+) diff --git a/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp b/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp index bee31513f1239..1602032216e1b 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp +++ b/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp @@ -40,6 +40,7 @@ int emscripten_builtin_pthread_create(void *thread, void *attr, void *(*callback)(void *), void *arg); int emscripten_builtin_pthread_join(void *th, void **ret); int emscripten_builtin_pthread_detach(void *th); +void emscripten_builtin_pthread_exit(void *th); } #endif diff --git a/system/lib/pthread/library_pthread_stub.c b/system/lib/pthread/library_pthread_stub.c index 0e016d2995c6e..bdd3934f0caac 100644 --- a/system/lib/pthread/library_pthread_stub.c +++ b/system/lib/pthread/library_pthread_stub.c @@ -222,7 +222,9 @@ _Noreturn void __pthread_exit(void* status) { exit(0); } +weak_alias(__pthread_exit, emscripten_builtin_pthread_exit); weak_alias(__pthread_exit, pthread_exit); +weak_alias(__pthread_exit, thrd_exit); int __pthread_detach(pthread_t t) { return 0; diff --git a/system/lib/pthread/pthread_create.c b/system/lib/pthread/pthread_create.c index 25ab956b4ce39..d7050bbc75516 100644 --- a/system/lib/pthread/pthread_create.c +++ b/system/lib/pthread/pthread_create.c @@ -361,4 +361,5 @@ _Noreturn void __pthread_exit(void* retval) { weak_alias(__pthread_create, emscripten_builtin_pthread_create); weak_alias(__pthread_create, pthread_create); +weak_alias(__pthread_exit, emscripten_builtin_pthread_exit); weak_alias(__pthread_exit, pthread_exit); From 19f4a18b02e07bc64cf7a33782e6a584c1f60673 Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Fri, 3 Nov 2023 16:13:12 -0700 Subject: [PATCH 07/10] Don't run internal_mprotect in Emscripten `__sanitizer::internal_mprotect` symbol produces a link-time error in `MprotectReadWrite`, which was added in LLVM 17. While I am not very familiar with this part of the code, it looks we're already avoiding running it like these in the same file: https://github.com/emscripten-core/emscripten/blob/8ecbdb3fc694f659aadb85a00d80777b20477281/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp#L148-L152 https://github.com/emscripten-core/emscripten/blob/8ecbdb3fc694f659aadb85a00d80777b20477281/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp#L156-L160 This does the same thing for `MprotectReadWrite`. --- .../lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp index 8159bc0b92c10..ae8f8fe07c588 100644 --- a/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp +++ b/system/lib/compiler-rt/lib/sanitizer_common/sanitizer_posix.cpp @@ -159,7 +159,11 @@ bool MprotectReadOnly(uptr addr, uptr size) { } bool MprotectReadWrite(uptr addr, uptr size) { +#if SANITIZER_EMSCRIPTEN + return true; +#else return 0 == internal_mprotect((void *)addr, size, PROT_READ | PROT_WRITE); +#endif } #if !SANITIZER_APPLE From 1e38f33acea6714c5129299389b9f7221c3564a5 Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Wed, 8 Nov 2023 14:01:01 -0800 Subject: [PATCH 08/10] Call pthread_attr_getdetachstate after pthread_attr_init In LLVM 16, in `pthread_create` LSan interceptor, if `attr` is NULL, it calls `pthread_attr_init` and initializes the `attr` with it, and then calls `pthread_attr_getdetachstate`: https://github.com/llvm/llvm-project/blob/7cbf1a2591520c2491aa35339f227775f4d3adf6/compiler-rt/lib/lsan/lsan_interceptors.cpp#L450-L456 In Emscripten, #15099 changes the `if` condition so that even if `attr` is not NULL, if it is `__ATTRP_C11_THREAD`, we call `pthread_attr_init`. `__ATTRP_C11_THREAD` looks like something used from musl, and is defined as -1. https://github.com/emscripten-core/emscripten/blob/5ce75b8828e3f50494c956d42f2bac301e41253b/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp#L465 In LLVM 17, somehow the order of the `pthread_attr_init` and `pthread_attr_getdetachstate` has swapped: https://github.com/llvm/llvm-project/blob/309d55140c46384b6de7a7573206cbeba3f7077f/compiler-rt/lib/lsan/lsan_interceptors.cpp#L444-L453 So we don't get to call `pthread_attr_init` before calling `pthread_attr_getdetachstate`. Even if the new code calls `pthread_attr_getdetachstate` only when `attr` is not NULL, in our case it didn't help because our `attr` was not NULL but `__ATTRP_C11_THREAD`. This swaps the code order back to what it was in LLVM 16. This is necessary to pass `lsan.test_pthread_c11_threads*`. Drive-by fix: This also guards `if (!attr || attr == __ATTRP_C11_THREAD)` condition with `SANITIZER_EMSCRIPTEN`, which was an Emscripten-specific fix added before. --- .../compiler-rt/lib/lsan/lsan_interceptors.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp b/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp index 1602032216e1b..216fb420d80f4 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp +++ b/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp @@ -453,17 +453,32 @@ INTERCEPTOR(int, pthread_create, void *th, void *attr, ENSURE_LSAN_INITED; EnsureMainThreadIDIsCorrect(); +#if !SANITIZER_EMSCRIPTEN bool detached = [attr]() { int d = 0; return attr && !pthread_attr_getdetachstate(attr, &d) && IsStateDetached(d); }(); +#endif __sanitizer_pthread_attr_t myattr; +#if SANITIZER_EMSCRIPTEN if (!attr || attr == __ATTRP_C11_THREAD) { +#else + if (!attr) { +#endif pthread_attr_init(&myattr); attr = &myattr; } AdjustStackSize(attr); +#if SANITIZER_EMSCRIPTEN + // In Emscripten sanitizer, attr can be nonzero but __ATTRP_C11_THREAD, in + // which case pthread_attr_getdetachstate needs to run after pthread_attr_init + // above. + bool detached = [attr]() { + int d = 0; + return attr && !pthread_attr_getdetachstate(attr, &d) && IsStateDetached(d); + }(); +#endif uptr this_tid = GetCurrentThreadId(); int result; { From f1cf73f0c13f8401770b42d5328d7fe61e099b0b Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Mon, 13 Nov 2023 21:54:10 -0800 Subject: [PATCH 09/10] Update readme.txt --- system/lib/compiler-rt/readme.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/system/lib/compiler-rt/readme.txt b/system/lib/compiler-rt/readme.txt index 4f1a788133218..d30655a776765 100644 --- a/system/lib/compiler-rt/readme.txt +++ b/system/lib/compiler-rt/readme.txt @@ -1,14 +1,14 @@ llvm's compiler-rt ------------------ -These files are from the llvm-project based on release 16.0.6. +These files are from the llvm-project based on release 17.0.4. We maintain a local fork of llvm-project that contains any emscripten specific patches: https://github.com/emscripten-core/llvm-project -The current patch is based on the emscripten-libs-16 branch. +The current patch is based on the emscripten-libs-17 branch. Update Instructions ------------------- @@ -20,4 +20,4 @@ Modifications For a list of changes from upstream see the compiler-rt files that are part of: -https://github.com/llvm/llvm-project/compare/llvmorg-16.0.6...emscripten-core:emscripten-libs-16 +https://github.com/llvm/llvm-project/compare/llvmorg-17.0.4...emscripten-core:emscripten-libs-17 From 9254a85fd17301eafce920923d00a758f98351cd Mon Sep 17 00:00:00 2001 From: Heejin Ahn Date: Thu, 16 Nov 2023 11:20:20 -0800 Subject: [PATCH 10/10] Address a comment Addressed https://github.com/emscripten-core/emscripten/pull/20708#discussion_r1396177861. --- .../lib/lsan/lsan_interceptors.cpp | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp b/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp index 216fb420d80f4..c4c6088d3663b 100644 --- a/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp +++ b/system/lib/compiler-rt/lib/lsan/lsan_interceptors.cpp @@ -453,32 +453,25 @@ INTERCEPTOR(int, pthread_create, void *th, void *attr, ENSURE_LSAN_INITED; EnsureMainThreadIDIsCorrect(); -#if !SANITIZER_EMSCRIPTEN +#if SANITIZER_EMSCRIPTEN + // In Emscripten sanitizer, attr can be nonzero but __ATTRP_C11_THREAD in case + // of C11 threads, in which case we need to run pthread_attr_init as well, so + // we treat __ATTRP_C11_THREAD like the nullptr in this function. + if (attr == __ATTRP_C11_THREAD) + attr = nullptr; +#endif + bool detached = [attr]() { int d = 0; return attr && !pthread_attr_getdetachstate(attr, &d) && IsStateDetached(d); }(); -#endif __sanitizer_pthread_attr_t myattr; -#if SANITIZER_EMSCRIPTEN - if (!attr || attr == __ATTRP_C11_THREAD) { -#else if (!attr) { -#endif pthread_attr_init(&myattr); attr = &myattr; } AdjustStackSize(attr); -#if SANITIZER_EMSCRIPTEN - // In Emscripten sanitizer, attr can be nonzero but __ATTRP_C11_THREAD, in - // which case pthread_attr_getdetachstate needs to run after pthread_attr_init - // above. - bool detached = [attr]() { - int d = 0; - return attr && !pthread_attr_getdetachstate(attr, &d) && IsStateDetached(d); - }(); -#endif uptr this_tid = GetCurrentThreadId(); int result; {