fix(ci): stop JAVA_TOOL_OPTIONS from silently killing integration tests

.gitlab/dd-trace-integration/.gitlab-ci.yml

@@ -287,7 +287,12 @@ post-pr-comment:
   stage: integration-test
   tags: [ "arch:arm64" ]
   image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1
+  id_tokens:
+    DDOCTOSTS_ID_TOKEN:
+      aud: dd-octo-sts
   needs:
+    - job: prepare:start
+      artifacts: true
     - job: prepare-patched-agent
       artifacts: true
     - job: integration-test-x64-glibc
@@ -309,4 +314,34 @@ post-pr-comment:
   timeout: 5m
   script:
     - .gitlab/dd-trace-integration/post-pr-comment.sh integration-test-results
-  allow_failure: true
+
+# Loud failure signal for scheduled/main runs, which have no PR to report back to.
+notify-slack-dd-trace-integration:
+  stage: notify
+  tags: [ "arch:amd64" ]
+  image: registry.ddbuild.io/slack-notifier:v91289620-4ec922a-latest@sha256:06b24f392ccc383d371c72001520a254edef523bc0bfdc445f487106107b4202
+  needs:
+    - job: prepare:start
+      artifacts: true
+    - job: get-versions
+      artifacts: true
+    - job: prepare-patched-agent
+      artifacts: false
+    - job: integration-test-x64-glibc
+      artifacts: false
+    - job: integration-test-x64-musl
+      artifacts: false
+    - job: integration-test-arm64-glibc
+      artifacts: false
+    - job: integration-test-arm64-musl
+      artifacts: false
+  rules:
+    - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null'
+      when: never
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      when: never
+    - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      when: on_failure
+    - when: never
+  script:
+    - .gitlab/dd-trace-integration/notify_channel.sh "${CURRENT_VERSION}"

.gitlab/dd-trace-integration/install-prerequisites.sh

@@ -54,14 +54,14 @@ if ! command -v jbang &> /dev/null; then
 
     # Verify installation
     if command -v jbang &> /dev/null; then
-        JBANG_VERSION=$(jbang version 2>&1 | head -1)
+        JBANG_VERSION=$(jbang version 2>&1 | head -1 || true)
         log_info "jbang installed successfully: ${JBANG_VERSION}"
     else
         log_warn "jbang installation completed but not found in PATH"
         log_warn "Please ensure ~/.jbang/bin is in your PATH"
     fi
 else
-    JBANG_VERSION=$(jbang version 2>&1 | head -1)
+    JBANG_VERSION=$(jbang version 2>&1 | head -1 || true)
     log_info "jbang already installed: ${JBANG_VERSION}"
 fi
 
@@ -124,7 +124,7 @@ else
             if [ -x "$JBANG_JDK_DIR/bin/java" ]; then
                 JDK25_INSTALLED=true
                 log_info "JDK 25 installed manually from Adoptium"
-                "$JBANG_JDK_DIR/bin/java" -version 2>&1 | head -1
+                "$JBANG_JDK_DIR/bin/java" -version 2>&1 | head -1 || true
             else
                 log_warn "JDK 25 extraction failed"
             fi
@@ -141,20 +141,33 @@ if [ "$JDK25_INSTALLED" = "false" ]; then
 fi
 
 # ========================================
-# Pre-warm jfr-shell backend
+# Pre-warm jfr-shell
 # ========================================
-# jafar-shell resolves its backend plugin (io.btrace:jfr-shell-jafar) from Maven at
-# runtime. If that artifact is unavailable (network restriction, version not yet
-# published), every validation run fails. Detect this early so we can skip gracefully.
+# jafar-shell itself (io.btrace:jafar-shell) is resolved by jbang from Maven Central,
+# which CI runners can't reach directly. Route jbang through the same internal Maven
+# proxy Gradle already uses (MAVEN_REPOSITORY_PROXY) so resolution doesn't depend on
+# public internet egress.
+#
+# jafar-shell also has an optional pluggable "backend" system (io.btrace:jfr-shell-jafar
+# / jfr-shell-jdk) resolved separately at runtime via its own embedded resolver, which
+# does not go through jbang's --repos and so cannot be routed through the proxy. That's
+# fine: the "open"+"show" flow used by validate-jfr-conformance.sh reads JFR files via
+# jafar-shell's own bundled parser and never touches the backend system, so a missing
+# backend does not affect real validation and must not be treated as fatal here.
 if [ ! -f /tmp/skip-jfr-validation ] && command -v jbang &> /dev/null; then
-    log_info "Pre-warming jfr-shell backend..."
-    PREWARM_OUT=$(jbang --java 25 jfr-shell@btraceio script /dev/null 2>&1 || true)
-    if echo "$PREWARM_OUT" | grep -q "No backends found\|No JFR backends available\|Failed to resolve artifact.*jfr-shell-jafar"; then
-        log_warn "jfr-shell backend unavailable (io.btrace:jfr-shell-jafar not resolvable from Maven)"
-        log_warn "JFR validation will be skipped"
-        echo "jfr-shell backend unavailable (io.btrace:jfr-shell-jafar not resolvable from Maven)" > /tmp/skip-jfr-validation
+    log_info "Pre-warming jfr-shell..."
+    if [ -n "${MAVEN_REPOSITORY_PROXY:-}" ]; then
+        JBANG_REPOS_OPT="--repos=central=${MAVEN_REPOSITORY_PROXY}"
     else
-        log_info "jfr-shell backend ready"
+        JBANG_REPOS_OPT=""
+    fi
+    if PREWARM_OUT=$(jbang --java 25 ${JBANG_REPOS_OPT} jfr-shell@btraceio script /dev/null 2>&1); then
+        log_info "jfr-shell ready"
+    else
+        log_warn "jfr-shell failed to start (io.btrace:jafar-shell not resolvable from Maven)"
+        log_warn "JFR validation will be skipped"
+        echo "jfr-shell failed to start (io.btrace:jafar-shell not resolvable from Maven)" > /tmp/skip-jfr-validation
+        echo "$PREWARM_OUT" | tail -20
     fi
 fi
 
@@ -164,7 +177,7 @@ fi
 if [ -z "${JAVA_HOME:-}" ]; then
     if command -v java &> /dev/null; then
         log_info "Java found in PATH"
-        java -version 2>&1 | head -3
+        java -version 2>&1 | head -3 || true
     else
         echo "ERROR: Java not found. Please set JAVA_HOME or ensure java is in PATH"
         exit 1
@@ -176,7 +189,7 @@ else
     fi
 
     log_info "Java found at JAVA_HOME: ${JAVA_HOME}"
-    "${JAVA_HOME}/bin/java" -version 2>&1 | head -3
+    "${JAVA_HOME}/bin/java" -version 2>&1 | head -3 || true
 fi
 
 # ========================================

.gitlab/dd-trace-integration/post-pr-comment.sh

@@ -13,10 +13,12 @@
 # Requires:
 # - DDPROF_COMMIT_BRANCH: Branch name to find PR
 # - CI_PIPELINE_URL: Link to pipeline
-# - pr-commenter tool (available in CI images)
+# - dd-octo-sts CLI (for GitHub token exchange, via upsert-github-pr-comment.sh)
 
 set -euo pipefail
 
+HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
 # Colors for logging
 GREEN='\033[0;32m'
 YELLOW='\033[1;33m'
@@ -28,55 +30,11 @@ log_warn() { echo -e "${YELLOW}[WARN]${NC} $*" >&2; }
 log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; }
 
 RESULTS_DIR="${1:-integration-test-results}"
-REPO="DataDog/java-profiler"
 
 # Dashboard URL (GitHub Pages)
 DASHBOARD_URL="https://datadog.github.io/java-profiler/integration/"
 
-# Check required tools - try to get pr-commenter from benchmarking-platform if not available
-PR_COMMENTER_AVAILABLE=false
-if command -v pr-commenter >/dev/null 2>&1; then
-  PR_COMMENTER_AVAILABLE=true
-elif [ -n "${CI_JOB_TOKEN:-}" ]; then
-  # In CI, clone benchmarking-platform to get pr-commenter
-  log_info "pr-commenter not found, cloning benchmarking-platform..."
-  PLATFORM_DIR=$(mktemp -d)
-  trap "rm -rf ${PLATFORM_DIR}" EXIT
-  git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/"
-  if git clone --depth 1 --branch dd-trace-go https://github.com/DataDog/benchmarking-platform "${PLATFORM_DIR}" 2>/dev/null; then
-    if [ -x "${PLATFORM_DIR}/tools/pr-commenter" ]; then
-      export PATH="${PLATFORM_DIR}/tools:${PATH}"
-      PR_COMMENTER_AVAILABLE=true
-      log_info "pr-commenter available from benchmarking-platform"
-    elif [ -f "${PLATFORM_DIR}/tools/pr-commenter.py" ]; then
-      # Try Python version
-      alias pr-commenter="python3 ${PLATFORM_DIR}/tools/pr-commenter.py"
-      PR_COMMENTER_AVAILABLE=true
-      log_info "pr-commenter.py available from benchmarking-platform"
-    else
-      log_warn "pr-commenter not found in benchmarking-platform"
-      ls -la "${PLATFORM_DIR}/tools/" 2>/dev/null || log_warn "No tools directory"
-    fi
-  else
-    log_warn "Failed to clone benchmarking-platform"
-  fi
-else
-  log_warn "pr-commenter not found and not in CI - will print comment instead"
-fi
-
-# Check required environment
-if [ -z "${DDPROF_COMMIT_BRANCH:-}" ]; then
-  log_warn "DDPROF_COMMIT_BRANCH not set - skipping comment"
-  exit 0
-fi
-
-# Skip for main/master branches (no PR)
-if [ "${DDPROF_COMMIT_BRANCH}" = "main" ] || [ "${DDPROF_COMMIT_BRANCH}" = "master" ]; then
-  log_info "Skipping PR comment for ${DDPROF_COMMIT_BRANCH} branch"
-  exit 0
-fi
-
-log_info "Posting comment for branch: ${DDPROF_COMMIT_BRANCH}"
+log_info "Collecting results for branch: ${DDPROF_COMMIT_BRANCH:-<unset>}"
 
 # Collect test results
 log_info "Collecting test results from ${RESULTS_DIR}..."
@@ -147,9 +105,12 @@ elif [ "${TOTAL_PASS}" -gt 0 ]; then
   STATUS_EMOJI=":white_check_mark:"
   STATUS_TEXT="PASSED"
 else
-  OVERALL_STATUS="neutral"
-  STATUS_EMOJI=":grey_question:"
-  STATUS_TEXT="NO RESULTS"
+  # No results at all usually means the test matrix never ran (setup/prerequisite
+  # failure) rather than a clean run — treat it as a failure so it isn't silently
+  # swallowed.
+  OVERALL_STATUS="failure"
+  STATUS_EMOJI=":rotating_light:"
+  STATUS_TEXT="COULD NOT RUN"
 fi
 
 log_info "Results: ${TOTAL_PASS} passed, ${TOTAL_FAIL} failed out of ${TOTAL} configurations"
@@ -162,6 +123,15 @@ if [ "${OVERALL_STATUS}" = "success" ]; then
   COMMENT_BODY=":white_check_mark: **All ${TOTAL} integration tests passed**
 
 :bar_chart: [Dashboard](${DASHBOARD_URL}) · :construction_worker: [Pipeline](${CI_PIPELINE_URL:-}) · :package: \`${DDPROF_SHA:0:8}\`"
+elif [ "${TOTAL}" -eq 0 ]; then
+  # No results produced at all - the test matrix itself never ran (e.g. a setup
+  # or prerequisite failure), not a clean pass/fail outcome.
+  COMMENT_BODY=":rotating_light: **Integration tests could not run — no results were produced**
+
+The test matrix in \`${RESULTS_DIR}\` is empty. This usually means a setup step
+(prerequisite installation, JDK provisioning, etc.) failed before any test could run.
+
+:construction_worker: [Pipeline](${CI_PIPELINE_URL:-}) · :package: \`${DDPROF_SHA:0:8}\`"
 else
   # Some failures or unknowns - show full matrix
   COMMENT_BODY="${STATUS_EMOJI} **${TOTAL_PASS}** passed, **${TOTAL_FAIL}** failed out of **${TOTAL}** configurations
@@ -205,27 +175,15 @@ $(echo -e "${FAILURES}")"
 - :package: Commit: \`${DDPROF_SHA}\`"
 fi
 
-# Post comment using pr-commenter
-if [ "${PR_COMMENTER_AVAILABLE}" = "true" ]; then
-  log_info "Posting comment via pr-commenter..."
-
-  if echo "${COMMENT_BODY}" | pr-commenter \
-      --for-repo="${REPO}" \
-      --for-pr="${DDPROF_COMMIT_BRANCH}" \
-      --header="Integration Tests" \
-      --on-duplicate=replace; then
-    log_info "Successfully posted comment"
-  else
-    log_error "Failed to post comment via pr-commenter"
-    log_info "Comment that would be posted:"
-    echo "${COMMENT_BODY}"
-    exit 1
-  fi
-else
-  log_info "Comment that would be posted to PR:"
-  echo ""
-  echo "${COMMENT_BODY}"
-  echo ""
+# Post comment via dd-octo-sts (upsert-github-pr-comment.sh handles missing
+# branch/PR/token gracefully, so a comment-posting problem never masks the
+# actual test outcome below).
+BODY_FILE=$(mktemp)
+trap 'rm -f "${BODY_FILE}"' EXIT
+echo "${COMMENT_BODY}" > "${BODY_FILE}"
+if ! "${HERE}/../scripts/upsert-github-pr-comment.sh" \
+    "dd-trace-integration-results" "${DDPROF_COMMIT_BRANCH:-}" "${BODY_FILE}"; then
+  log_error "Failed to post PR comment (transport failure) — continuing; only the test outcome below determines pipeline status"
 fi
 
 # Exit with failure if tests failed (makes pipeline fail)

.gitlab/dd-trace-integration/run-integration-test.sh

@@ -103,7 +103,7 @@ collect_system_metrics() {
   local cpu_count=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo "1")
   local cpu_quota=$(cat /sys/fs/cgroup/cpu/cpu.cfs_quota_us 2>/dev/null || echo "-1")
   local cpu_period=$(cat /sys/fs/cgroup/cpu/cpu.cfs_period_us 2>/dev/null || echo "-1")
-  local load_avg=$(uptime | awk -F'load average:' '{print $2}' | xargs)
+  local load_avg=$(command -v uptime >/dev/null 2>&1 && uptime | awk -F'load average:' '{print $2}' | xargs || echo "unavailable")
   local container=$(test -f /.dockerenv && echo "true" || echo "false")
 
   # Parse throttling stats
@@ -195,7 +195,7 @@ mkdir -p "${RESULTS_DIR}"
 
 log_info "Results directory: ${RESULTS_DIR}"
 log_info "Java version:"
-"${JAVA_HOME}/bin/java" -version 2>&1 | head -3
+"${JAVA_HOME}/bin/java" -version 2>&1 | head -3 || true
 
 # ========================================
 # Install Prerequisites
@@ -220,7 +220,7 @@ if ! command -v jbang &> /dev/null; then
   exit 1
 fi
 
-log_info "jbang version: $(jbang version 2>&1 | head -1)"
+log_info "jbang version: $(jbang version 2>&1 | head -1 || true)"
 
 # ========================================
 # Artifact Collection on Exit