Missing libgcc_s.so.1 on the host should not result in crash

ddprof-lib/src/main/cpp/faultInjection.h

@@ -28,6 +28,12 @@
 //   pc = SafeAccess::load(INJECT_FAULT_ADDRESS_LIKELY((void**)fp));
 //   VMMethod* m = ((VMMethod**)INJECT_FAULT_ADDRESS_UNLIKELY(fp))[off];
 //
+// INJECT_FAULT_BOOL_* wraps the *result* of a call that already ran for
+// real, forcing it to report `false` so a caller's failure-handling path
+// (not its memory-safety recovery path) gets exercised, e.g.:
+//
+//   return INJECT_FAULT_BOOL_LIKELY(dlopen(name, flags) != nullptr);
+//
 // The three tiers name their firing frequency: RARE 0.01%, UNLIKELY 0.1%,
 // LIKELY 1%.  See faultInjection.cpp for the poison-address and PRNG details.
 
@@ -76,6 +82,19 @@ inline T injectAddress(T ptr, u64 threshold, const char* fn) {
   }
   return ptr;
 }
+
+// Returns orig unchanged, or `faulty` when the tier fires. Unlike
+// injectAddress() (which fakes an input about to be dereferenced), this fakes
+// the *outcome* of a call that already ran for real — e.g. making a
+// successful dlopen() appear to have failed, to exercise a caller's error
+// path without needing the library to actually be absent.
+template <typename T>
+inline T injectValue(T orig, T faulty, u64 threshold, const char* fn) {
+  if (__builtin_expect(shouldFire(threshold, fn), 0)) {
+    return faulty;
+  }
+  return orig;
+}
 }  // namespace faultinj
 
 #define INJECT_FAULT_ADDRESS_RARE(ptr) \
@@ -85,6 +104,13 @@ inline T injectAddress(T ptr, u64 threshold, const char* fn) {
 #define INJECT_FAULT_ADDRESS_LIKELY(ptr) \
     ::faultinj::injectAddress((ptr), ::faultinj::PROB_LIKELY, __func__)
 
+#define INJECT_FAULT_BOOL_RARE(v) \
+    ::faultinj::injectValue((v), false, ::faultinj::PROB_RARE, __func__)
+#define INJECT_FAULT_BOOL_UNLIKELY(v) \
+    ::faultinj::injectValue((v), false, ::faultinj::PROB_UNLIKELY, __func__)
+#define INJECT_FAULT_BOOL_LIKELY(v) \
+    ::faultinj::injectValue((v), false, ::faultinj::PROB_LIKELY, __func__)
+
 #else  // __FAULT_INJECTION__ not defined — strict identity, zero cost.
 
 #define INJECT_FAULT_ADDRESS_RARE(ptr)     (ptr)
@@ -99,6 +125,10 @@ inline T injectAddress(T ptr, u64 threshold, const char* fn) {
 #define INJECT_FAULT_LONG_UNLIKELY(v) (v)
 #define INJECT_FAULT_LONG_LIKELY(v)   (v)
 
+#define INJECT_FAULT_BOOL_RARE(v)     (v)
+#define INJECT_FAULT_BOOL_UNLIKELY(v) (v)
+#define INJECT_FAULT_BOOL_LIKELY(v)   (v)
+
 #endif  // __FAULT_INJECTION__
 
 #endif  // _FAULT_INJECTION_H

ddprof-lib/src/main/cpp/profiler.cpp

@@ -843,7 +843,7 @@ void Profiler::writeHeapUsage(long value, bool live) {
   _locks[lock_index].unlock();
 }
 
-void Profiler::prewarmUnwinder() {
+bool Profiler::prewarmUnwinder() {
 #ifdef __linux__
   // J9 on aarch64 (and other JVMs) lazily loads libgcc_s.so.1 from its DWARF
   // unwinder during stack walks. When that happens inside a signal handler
@@ -864,7 +864,13 @@ void Profiler::prewarmUnwinder() {
   // dlopen by SONAME is the only mechanism that works under static-libgcc.
   // libgcc_s.so.1 has been the stable SONAME since 2002; a bump would
   // constitute a glibc/GCC C++ ABI break and is treated as a fixed contract.
-  (void)dlopen("libgcc_s.so.1", RTLD_LAZY | RTLD_GLOBAL);
+  //
+  // INJECT_FAULT_BOOL_LIKELY lets fault-injection builds force this to
+  // report failure without the library actually being absent, so
+  // checkState()'s "Missing libgcc_s.so" path can be exercised in CI.
+  return INJECT_FAULT_BOOL_LIKELY(dlopen("libgcc_s.so.1", RTLD_LAZY | RTLD_GLOBAL) != nullptr);
+#else
+  return true;
 #endif
 }
 
@@ -1291,6 +1297,13 @@ Error Profiler::checkState() {
   if (s == ERROR) {
     return Error("Profiler encountered fatal error");
   } else if (s == NEW) {
+    // Force libgcc_s to load now (idempotent dlopen) so the JVM's DWARF
+    // unwinder cannot lazy-load it later from signal context.
+    if (!prewarmUnwinder()) {
+      _state.store(ERROR, std::memory_order_release);
+      return Error("Missing libgcc_s.so.1");
+    }
+
     // Make sure JVMSupport is initialized
     // In theory, it should be initialized in JVMTI::VMInit() callback,
     // but the callback arrives too late, after this method is called.
@@ -1306,6 +1319,7 @@ Error Profiler::checkState() {
 
 Error Profiler::init() {
   MutexLocker ml(_state_lock);
+
   State s = state();
   if (s == ERROR) {
     return Error("Profiler encountered fatal error");
@@ -1336,10 +1350,6 @@ Error Profiler::start(Arguments &args, bool reset) {
     return error;
   }
 
-  // Force libgcc_s to load now (idempotent dlopen) so the JVM's DWARF
-  // unwinder cannot lazy-load it later from signal context.
-  prewarmUnwinder();
-
   error = checkJvmCapabilities();
   if (error) {
     return error;

ddprof-lib/src/main/cpp/profiler.h

@@ -152,7 +152,7 @@ class alignas(alignof(SpinLock)) Profiler {
   void **_dlopen_entry;
   static void *dlopen_hook(const char *filename, int flags);
   void switchLibraryTrap(bool enable);
-  static void prewarmUnwinder();
+  static bool prewarmUnwinder();
 
   void disableEngines();
 

ddprof-lib/src/test/cpp/faultInjection_ut.cpp

@@ -10,10 +10,14 @@
 #include <sys/mman.h>
 #include <unistd.h>
 
+#include <cstring>
+
 #include "faultInjection.h"
 #include "safeAccess.h"
 #include "os.h"
+#include "profiler.h"
 #include "threadLocalData.h"
+#include "vmEntry.h"
 #include "hotspot/hotspotSupport.h"
 #include "../../main/cpp/gtest_crash_handler.h"
 
@@ -47,6 +51,18 @@ TEST(FaultInjectionTest, DisabledValueMacrosAreIdentity) {
   EXPECT_EQ(INJECT_FAULT_LONG_RARE(l), l);
   EXPECT_EQ(INJECT_FAULT_LONG_UNLIKELY(l), l);
   EXPECT_EQ(INJECT_FAULT_LONG_LIKELY(l), l);
+
+  // BOOL must be identity both ways -- an accidental non-identity expansion
+  // (e.g. always forcing false) would otherwise only show up as a silent
+  // behavioural change in a production build, never a compile error.
+  bool t = true;
+  bool f = false;
+  EXPECT_EQ(INJECT_FAULT_BOOL_RARE(t), t);
+  EXPECT_EQ(INJECT_FAULT_BOOL_UNLIKELY(t), t);
+  EXPECT_EQ(INJECT_FAULT_BOOL_LIKELY(t), t);
+  EXPECT_EQ(INJECT_FAULT_BOOL_RARE(f), f);
+  EXPECT_EQ(INJECT_FAULT_BOOL_UNLIKELY(f), f);
+  EXPECT_EQ(INJECT_FAULT_BOOL_LIKELY(f), f);
 }
 
 #else  // __FAULT_INJECTION__ enabled (built under -PenableFaultInjection)
@@ -180,4 +196,98 @@ TEST_F(FaultInjectionTest, WalkVmSigsetjmpRecoversFromInjectedFault) {
   SUCCEED();
 }
 
+// Friend of Profiler (see profiler.h) — lets this test force the internal
+// state to a known value so checkState() can be exercised deterministically
+// (matches the pattern in jvmSupport_ut.cpp).
+class ProfilerTestAccessor {
+public:
+  static void setState(Profiler* p, State s) {
+    p->_state.store(s, std::memory_order_release);
+  }
+  static State getState(Profiler* p) {
+    return p->_state.load(std::memory_order_acquire);
+  }
+};
+
+// Friend of VM (see vmEntry.h) — lets this test install a mock jvmtiEnv, the
+// same seam jvmSupport_ut.cpp uses. checkState() (below) checks
+// prewarmUnwinder() before JVMSupport::initialize(), so the injected-failure
+// path never touches this at all; it exists only so the ~99% non-injected
+// iterations, which do fall through into JVMSupport::initialize(), fail
+// gracefully instead of crashing on a null VM::_jvmti in this no-live-JVM
+// binary. Unlike a JVMThread-level fake (which would permanently flip
+// JVMThread::isInitialized() for the rest of the process, since ThreadLocal
+// pthread keys are never invalidated), this is a plain pointer swap that
+// ScopedJvmtiMock restores on scope exit -- no state leaks into later tests.
+class VMTestAccessor {
+public:
+  static jvmtiEnv* getJvmti() { return VM::_jvmti; }
+  static void setJvmti(jvmtiEnv* env) { VM::_jvmti = env; }
+};
+
+static jvmtiError JNICALL mock_GetCurrentThread_fails(jvmtiEnv*, jthread*) {
+  return JVMTI_ERROR_INTERNAL;
+}
+
+class ScopedJvmtiMock {
+public:
+  ScopedJvmtiMock() : _orig(VMTestAccessor::getJvmti()) {
+    _tbl.GetCurrentThread = &mock_GetCurrentThread_fails;
+    _env.functions = &_tbl;
+    VMTestAccessor::setJvmti(&_env);
+  }
+  ~ScopedJvmtiMock() { VMTestAccessor::setJvmti(_orig); }
+
+private:
+  jvmtiInterface_1_ _tbl{};
+  _jvmtiEnv _env{};
+  jvmtiEnv* _orig;
+};
+
+// (d) Value-injection path: PROF-15395 fixed Profiler::checkState() (shared by
+// start()/check(), and therefore also reached by the -agentpath auto-start
+// path) to fail cleanly instead of crashing later when libgcc_s.so.1 can't be
+// loaded. libgcc_s.so.1 is always present in this test environment, so
+// INJECT_FAULT_BOOL_LIKELY on prewarmUnwinder()'s return value is what makes
+// that failure path reachable here: the real dlopen() still runs and
+// succeeds, but the caller is deterministically told it failed.
+TEST_F(FaultInjectionTest, CheckStateSurfacesInjectedPrewarmUnwinderFailure) {
+#ifdef __linux__
+  Profiler* p = Profiler::instance();
+  // checkState() checks prewarmUnwinder() before JVMSupport::initialize(), so
+  // reaching the injected-failure path below needs nothing but the NEW state.
+  ScopedJvmtiMock jvmti_mock;
+  ProfilerTestAccessor::setState(p, NEW);
+  ProfiledThread::current()->setFiRng(0x5EED5EED5EED5EEDULL);
+
+  bool sawInjectedFailure = false;
+  bool sawNonInjectedPrewarm = false;
+  // shouldFire() mixes the fixed RNG seed above with an ASLR-dependent
+  // per-call-site address, so which outcome the *first* call produces is not
+  // deterministic run to run -- the injected failure can land before a
+  // non-injected call is observed. Keep iterating (and un-latching the ERROR
+  // state that every outcome here leaves behind) until both have been seen.
+  for (int i = 0; i < 5000 && !(sawInjectedFailure && sawNonInjectedPrewarm); i++) {
+    Error error = p->checkState();
+    ASSERT_TRUE((bool)error) << "checkState() must fail here: either the "
+                                 "injected prewarmUnwinder() failure or the "
+                                 "mocked JVMSupport::initialize() failure";
+    if (std::strcmp(error.message(), "Missing libgcc_s.so.1") == 0) {
+      sawInjectedFailure = true;
+    } else {
+      // prewarmUnwinder() succeeded (non-injected, ~99% of calls) and fell
+      // through to the mocked JVMSupport::initialize() failure instead.
+      EXPECT_STREQ("Profiler encountered fatal error", error.message());
+      sawNonInjectedPrewarm = true;
+    }
+    ProfilerTestAccessor::setState(p, NEW);
+  }
+
+  EXPECT_TRUE(sawInjectedFailure)
+      << "expected at least one injected prewarmUnwinder() failure within 5000 tries";
+  EXPECT_TRUE(sawNonInjectedPrewarm)
+      << "expected at least one non-injected prewarmUnwinder() success within 5000 tries";
+#endif // __linux__
+}
+
 #endif  // __FAULT_INJECTION__

ddprof-lib/src/test/cpp/jvmSupport_ut.cpp

@@ -4,6 +4,7 @@
  */
 
 #include <gtest/gtest.h>
+#include <cstring>
 #include "jvmSupport.h"
 #include "jvmThread.h"
 #include "vmEntry.h"
@@ -111,7 +112,23 @@ TEST_F(JvmSupportInitFailureTest, JVMSupportInitializeFailsWhenJVMThreadFails) {
 TEST_F(JvmSupportInitFailureTest, CheckStateBlocksOnInitFailureAndLatchesError) {
     Profiler* p = Profiler::instance();
 
-    Error error = p->checkState();
+    // Under -PenableFaultInjection, checkState() checks prewarmUnwinder()
+    // before JVMSupport::initialize() (see profiler.cpp), so an injected
+    // fault could occasionally surface "Missing libgcc_s.so" here instead of
+    // the JVMSupport::initialize() failure this test targets. Retry past any
+    // such injected failure. If it persists across retries, libgcc_s is likely
+    // genuinely absent on this host and this test cannot exercise the intended path.
+    Error error = Error::OK;
+    for (int i = 0; i < 100; i++) {
+        error = p->checkState();
+        if (!error || std::strcmp(error.message(), "Missing libgcc_s.so.1") != 0) {
+            break;
+        }
+        ProfilerTestAccessor::setState(p, NEW);
+    }
+    if (error && std::strcmp(error.message(), "Missing libgcc_s.so.1") == 0) {
+        GTEST_SKIP() << "libgcc_s.so.1 is missing on this host; cannot exercise JVMSupport::initialize() failure path";
+    }
     bool has_error = (bool)error;
 
     EXPECT_TRUE(has_error);