diff --git a/ddprof-lib/src/main/cpp/faultInjection.h b/ddprof-lib/src/main/cpp/faultInjection.h index e19f4bb1e..a88dbe5c4 100644 --- a/ddprof-lib/src/main/cpp/faultInjection.h +++ b/ddprof-lib/src/main/cpp/faultInjection.h @@ -28,6 +28,12 @@ // pc = SafeAccess::load(INJECT_FAULT_ADDRESS_LIKELY((void**)fp)); // VMMethod* m = ((VMMethod**)INJECT_FAULT_ADDRESS_UNLIKELY(fp))[off]; // +// INJECT_FAULT_BOOL_* wraps the *result* of a call that already ran for +// real, forcing it to report `false` so a caller's failure-handling path +// (not its memory-safety recovery path) gets exercised, e.g.: +// +// return INJECT_FAULT_BOOL_LIKELY(dlopen(name, flags) != nullptr); +// // The three tiers name their firing frequency: RARE 0.01%, UNLIKELY 0.1%, // LIKELY 1%. See faultInjection.cpp for the poison-address and PRNG details. @@ -76,6 +82,19 @@ inline T injectAddress(T ptr, u64 threshold, const char* fn) { } return ptr; } + +// Returns orig unchanged, or `faulty` when the tier fires. Unlike +// injectAddress() (which fakes an input about to be dereferenced), this fakes +// the *outcome* of a call that already ran for real — e.g. making a +// successful dlopen() appear to have failed, to exercise a caller's error +// path without needing the library to actually be absent. +template +inline T injectValue(T orig, T faulty, u64 threshold, const char* fn) { + if (__builtin_expect(shouldFire(threshold, fn), 0)) { + return faulty; + } + return orig; +} } // namespace faultinj #define INJECT_FAULT_ADDRESS_RARE(ptr) \ @@ -85,6 +104,13 @@ inline T injectAddress(T ptr, u64 threshold, const char* fn) { #define INJECT_FAULT_ADDRESS_LIKELY(ptr) \ ::faultinj::injectAddress((ptr), ::faultinj::PROB_LIKELY, __func__) +#define INJECT_FAULT_BOOL_RARE(v) \ + ::faultinj::injectValue((v), false, ::faultinj::PROB_RARE, __func__) +#define INJECT_FAULT_BOOL_UNLIKELY(v) \ + ::faultinj::injectValue((v), false, ::faultinj::PROB_UNLIKELY, __func__) +#define INJECT_FAULT_BOOL_LIKELY(v) \ + ::faultinj::injectValue((v), false, ::faultinj::PROB_LIKELY, __func__) + #else // __FAULT_INJECTION__ not defined — strict identity, zero cost. #define INJECT_FAULT_ADDRESS_RARE(ptr) (ptr) @@ -99,6 +125,10 @@ inline T injectAddress(T ptr, u64 threshold, const char* fn) { #define INJECT_FAULT_LONG_UNLIKELY(v) (v) #define INJECT_FAULT_LONG_LIKELY(v) (v) +#define INJECT_FAULT_BOOL_RARE(v) (v) +#define INJECT_FAULT_BOOL_UNLIKELY(v) (v) +#define INJECT_FAULT_BOOL_LIKELY(v) (v) + #endif // __FAULT_INJECTION__ #endif // _FAULT_INJECTION_H diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp index 7c631b002..32ff6e21c 100644 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ b/ddprof-lib/src/main/cpp/profiler.cpp @@ -843,7 +843,7 @@ void Profiler::writeHeapUsage(long value, bool live) { _locks[lock_index].unlock(); } -void Profiler::prewarmUnwinder() { +bool Profiler::prewarmUnwinder() { #ifdef __linux__ // J9 on aarch64 (and other JVMs) lazily loads libgcc_s.so.1 from its DWARF // unwinder during stack walks. When that happens inside a signal handler @@ -864,7 +864,13 @@ void Profiler::prewarmUnwinder() { // dlopen by SONAME is the only mechanism that works under static-libgcc. // libgcc_s.so.1 has been the stable SONAME since 2002; a bump would // constitute a glibc/GCC C++ ABI break and is treated as a fixed contract. - (void)dlopen("libgcc_s.so.1", RTLD_LAZY | RTLD_GLOBAL); + // + // INJECT_FAULT_BOOL_LIKELY lets fault-injection builds force this to + // report failure without the library actually being absent, so + // checkState()'s "Missing libgcc_s.so" path can be exercised in CI. + return INJECT_FAULT_BOOL_LIKELY(dlopen("libgcc_s.so.1", RTLD_LAZY | RTLD_GLOBAL) != nullptr); +#else + return true; #endif } @@ -1291,6 +1297,13 @@ Error Profiler::checkState() { if (s == ERROR) { return Error("Profiler encountered fatal error"); } else if (s == NEW) { + // Force libgcc_s to load now (idempotent dlopen) so the JVM's DWARF + // unwinder cannot lazy-load it later from signal context. + if (!prewarmUnwinder()) { + _state.store(ERROR, std::memory_order_release); + return Error("Missing libgcc_s.so.1"); + } + // Make sure JVMSupport is initialized // In theory, it should be initialized in JVMTI::VMInit() callback, // but the callback arrives too late, after this method is called. @@ -1306,6 +1319,7 @@ Error Profiler::checkState() { Error Profiler::init() { MutexLocker ml(_state_lock); + State s = state(); if (s == ERROR) { return Error("Profiler encountered fatal error"); @@ -1336,10 +1350,6 @@ Error Profiler::start(Arguments &args, bool reset) { return error; } - // Force libgcc_s to load now (idempotent dlopen) so the JVM's DWARF - // unwinder cannot lazy-load it later from signal context. - prewarmUnwinder(); - error = checkJvmCapabilities(); if (error) { return error; diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h index 946c4c1e6..3e4ffffce 100644 --- a/ddprof-lib/src/main/cpp/profiler.h +++ b/ddprof-lib/src/main/cpp/profiler.h @@ -152,7 +152,7 @@ class alignas(alignof(SpinLock)) Profiler { void **_dlopen_entry; static void *dlopen_hook(const char *filename, int flags); void switchLibraryTrap(bool enable); - static void prewarmUnwinder(); + static bool prewarmUnwinder(); void disableEngines(); diff --git a/ddprof-lib/src/test/cpp/faultInjection_ut.cpp b/ddprof-lib/src/test/cpp/faultInjection_ut.cpp index d1d25c250..e3d94f8e1 100644 --- a/ddprof-lib/src/test/cpp/faultInjection_ut.cpp +++ b/ddprof-lib/src/test/cpp/faultInjection_ut.cpp @@ -10,10 +10,14 @@ #include #include +#include + #include "faultInjection.h" #include "safeAccess.h" #include "os.h" +#include "profiler.h" #include "threadLocalData.h" +#include "vmEntry.h" #include "hotspot/hotspotSupport.h" #include "../../main/cpp/gtest_crash_handler.h" @@ -47,6 +51,18 @@ TEST(FaultInjectionTest, DisabledValueMacrosAreIdentity) { EXPECT_EQ(INJECT_FAULT_LONG_RARE(l), l); EXPECT_EQ(INJECT_FAULT_LONG_UNLIKELY(l), l); EXPECT_EQ(INJECT_FAULT_LONG_LIKELY(l), l); + + // BOOL must be identity both ways -- an accidental non-identity expansion + // (e.g. always forcing false) would otherwise only show up as a silent + // behavioural change in a production build, never a compile error. + bool t = true; + bool f = false; + EXPECT_EQ(INJECT_FAULT_BOOL_RARE(t), t); + EXPECT_EQ(INJECT_FAULT_BOOL_UNLIKELY(t), t); + EXPECT_EQ(INJECT_FAULT_BOOL_LIKELY(t), t); + EXPECT_EQ(INJECT_FAULT_BOOL_RARE(f), f); + EXPECT_EQ(INJECT_FAULT_BOOL_UNLIKELY(f), f); + EXPECT_EQ(INJECT_FAULT_BOOL_LIKELY(f), f); } #else // __FAULT_INJECTION__ enabled (built under -PenableFaultInjection) @@ -180,4 +196,98 @@ TEST_F(FaultInjectionTest, WalkVmSigsetjmpRecoversFromInjectedFault) { SUCCEED(); } +// Friend of Profiler (see profiler.h) — lets this test force the internal +// state to a known value so checkState() can be exercised deterministically +// (matches the pattern in jvmSupport_ut.cpp). +class ProfilerTestAccessor { +public: + static void setState(Profiler* p, State s) { + p->_state.store(s, std::memory_order_release); + } + static State getState(Profiler* p) { + return p->_state.load(std::memory_order_acquire); + } +}; + +// Friend of VM (see vmEntry.h) — lets this test install a mock jvmtiEnv, the +// same seam jvmSupport_ut.cpp uses. checkState() (below) checks +// prewarmUnwinder() before JVMSupport::initialize(), so the injected-failure +// path never touches this at all; it exists only so the ~99% non-injected +// iterations, which do fall through into JVMSupport::initialize(), fail +// gracefully instead of crashing on a null VM::_jvmti in this no-live-JVM +// binary. Unlike a JVMThread-level fake (which would permanently flip +// JVMThread::isInitialized() for the rest of the process, since ThreadLocal +// pthread keys are never invalidated), this is a plain pointer swap that +// ScopedJvmtiMock restores on scope exit -- no state leaks into later tests. +class VMTestAccessor { +public: + static jvmtiEnv* getJvmti() { return VM::_jvmti; } + static void setJvmti(jvmtiEnv* env) { VM::_jvmti = env; } +}; + +static jvmtiError JNICALL mock_GetCurrentThread_fails(jvmtiEnv*, jthread*) { + return JVMTI_ERROR_INTERNAL; +} + +class ScopedJvmtiMock { +public: + ScopedJvmtiMock() : _orig(VMTestAccessor::getJvmti()) { + _tbl.GetCurrentThread = &mock_GetCurrentThread_fails; + _env.functions = &_tbl; + VMTestAccessor::setJvmti(&_env); + } + ~ScopedJvmtiMock() { VMTestAccessor::setJvmti(_orig); } + +private: + jvmtiInterface_1_ _tbl{}; + _jvmtiEnv _env{}; + jvmtiEnv* _orig; +}; + +// (d) Value-injection path: PROF-15395 fixed Profiler::checkState() (shared by +// start()/check(), and therefore also reached by the -agentpath auto-start +// path) to fail cleanly instead of crashing later when libgcc_s.so.1 can't be +// loaded. libgcc_s.so.1 is always present in this test environment, so +// INJECT_FAULT_BOOL_LIKELY on prewarmUnwinder()'s return value is what makes +// that failure path reachable here: the real dlopen() still runs and +// succeeds, but the caller is deterministically told it failed. +TEST_F(FaultInjectionTest, CheckStateSurfacesInjectedPrewarmUnwinderFailure) { +#ifdef __linux__ + Profiler* p = Profiler::instance(); + // checkState() checks prewarmUnwinder() before JVMSupport::initialize(), so + // reaching the injected-failure path below needs nothing but the NEW state. + ScopedJvmtiMock jvmti_mock; + ProfilerTestAccessor::setState(p, NEW); + ProfiledThread::current()->setFiRng(0x5EED5EED5EED5EEDULL); + + bool sawInjectedFailure = false; + bool sawNonInjectedPrewarm = false; + // shouldFire() mixes the fixed RNG seed above with an ASLR-dependent + // per-call-site address, so which outcome the *first* call produces is not + // deterministic run to run -- the injected failure can land before a + // non-injected call is observed. Keep iterating (and un-latching the ERROR + // state that every outcome here leaves behind) until both have been seen. + for (int i = 0; i < 5000 && !(sawInjectedFailure && sawNonInjectedPrewarm); i++) { + Error error = p->checkState(); + ASSERT_TRUE((bool)error) << "checkState() must fail here: either the " + "injected prewarmUnwinder() failure or the " + "mocked JVMSupport::initialize() failure"; + if (std::strcmp(error.message(), "Missing libgcc_s.so.1") == 0) { + sawInjectedFailure = true; + } else { + // prewarmUnwinder() succeeded (non-injected, ~99% of calls) and fell + // through to the mocked JVMSupport::initialize() failure instead. + EXPECT_STREQ("Profiler encountered fatal error", error.message()); + sawNonInjectedPrewarm = true; + } + ProfilerTestAccessor::setState(p, NEW); + } + + EXPECT_TRUE(sawInjectedFailure) + << "expected at least one injected prewarmUnwinder() failure within 5000 tries"; + EXPECT_TRUE(sawNonInjectedPrewarm) + << "expected at least one non-injected prewarmUnwinder() success within 5000 tries"; +#endif // __linux__ +} + #endif // __FAULT_INJECTION__ diff --git a/ddprof-lib/src/test/cpp/jvmSupport_ut.cpp b/ddprof-lib/src/test/cpp/jvmSupport_ut.cpp index 6557567eb..c203d296e 100644 --- a/ddprof-lib/src/test/cpp/jvmSupport_ut.cpp +++ b/ddprof-lib/src/test/cpp/jvmSupport_ut.cpp @@ -4,6 +4,7 @@ */ #include +#include #include "jvmSupport.h" #include "jvmThread.h" #include "vmEntry.h" @@ -111,7 +112,23 @@ TEST_F(JvmSupportInitFailureTest, JVMSupportInitializeFailsWhenJVMThreadFails) { TEST_F(JvmSupportInitFailureTest, CheckStateBlocksOnInitFailureAndLatchesError) { Profiler* p = Profiler::instance(); - Error error = p->checkState(); + // Under -PenableFaultInjection, checkState() checks prewarmUnwinder() + // before JVMSupport::initialize() (see profiler.cpp), so an injected + // fault could occasionally surface "Missing libgcc_s.so" here instead of + // the JVMSupport::initialize() failure this test targets. Retry past any + // such injected failure. If it persists across retries, libgcc_s is likely + // genuinely absent on this host and this test cannot exercise the intended path. + Error error = Error::OK; + for (int i = 0; i < 100; i++) { + error = p->checkState(); + if (!error || std::strcmp(error.message(), "Missing libgcc_s.so.1") != 0) { + break; + } + ProfilerTestAccessor::setState(p, NEW); + } + if (error && std::strcmp(error.message(), "Missing libgcc_s.so.1") == 0) { + GTEST_SKIP() << "libgcc_s.so.1 is missing on this host; cannot exercise JVMSupport::initialize() failure path"; + } bool has_error = (bool)error; EXPECT_TRUE(has_error);