From 1b54c6385bafed2894d6c85f6cfbe0e005b9bf2d Mon Sep 17 00:00:00 2001 From: Brian Marks Date: Thu, 13 Aug 2026 11:17:16 -0400 Subject: [PATCH 1/2] ci: make request replayer concurrency safe --- .gitlab-ci.yml | 1 + .gitlab/generate-common.php | 17 +- .gitlab/install-request-replayer-source.sh | 19 ++ .gitlab/test-request-replayer-concurrency.php | 245 ++++++++++++++ .../services/request-replayer/src/index.php | 308 +++++++++--------- .../request-replayer/src/state-lock.php | 19 ++ 6 files changed, 461 insertions(+), 148 deletions(-) create mode 100644 .gitlab/install-request-replayer-source.sh create mode 100644 .gitlab/test-request-replayer-concurrency.php create mode 100644 dockerfiles/services/request-replayer/src/state-lock.php diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6a570f099a..fa801057cb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -52,6 +52,7 @@ generate-templates: LIBDATADOG_SHA=$(git ls-tree HEAD libdatadog | awk '{print $3}') printf 'LIBDATADOG_OVERRIDE_SHA=%s\nLIBDATADOG_FEEDBACK_RUN=true\n' "$LIBDATADOG_SHA" >> libdatadog.env fi + - php ./.gitlab/test-request-replayer-concurrency.php - php ./.gitlab/generate-package.php | tee .gitlab/package-gen.yml - php ./.gitlab/generate-tracer.php | tee .gitlab/tracer-gen.yml - php ./.gitlab/generate-appsec.php | tee .gitlab/appsec-gen.yml diff --git a/.gitlab/generate-common.php b/.gitlab/generate-common.php index 86678ba917..b2061fa03e 100644 --- a/.gitlab/generate-common.php +++ b/.gitlab/generate-common.php @@ -229,9 +229,24 @@ function windows_git_setup_with_packages() { request-replayer: name: registry.ddbuild.io/images/mirror/datadog/dd-trace-ci:php-request-replayer-2.0 alias: request-replayer - command: ["php", "-S", ":80", "index.php"] + command: + - sh + - -c + - | + source_dir="$CI_PROJECT_DIR/dockerfiles/services/request-replayer/src" + printf '%s\n' '<''?php while (!file_exists("/tmp/request-replayer-ready")) { usleep(100000); } require "/var/www/index.php";' > /tmp/request-replayer-router.php + ( + installer="$CI_PROJECT_DIR/.gitlab/install-request-replayer-source.sh" + while [ ! -f "$installer" ]; do sleep 1; done + sh "$installer" "$source_dir" /var/www /tmp/request-replayer-ready + ) & + (while true; do php /var/www/metricsserver.php; sleep 1; done) & + echo "$!" > /tmp/metrics-server.pid + exec php -S :80 /tmp/request-replayer-router.php variables: DD_REQUEST_DUMPER_FILE: dump.json + PHP_CLI_SERVER_WORKERS: "16" + REQUEST_REPLAYER_METRICS_SERVER_MANAGED: "1" KUBERNETES_SERVICE_CPU_REQUEST: 2 KUBERNETES_SERVICE_CPU_LIMIT: 2 KUBERNETES_SERVICE_MEMORY_REQUEST: 1Gi diff --git a/.gitlab/install-request-replayer-source.sh b/.gitlab/install-request-replayer-source.sh new file mode 100644 index 0000000000..14654ede78 --- /dev/null +++ b/.gitlab/install-request-replayer-source.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +set -u + +source_dir=$1 +target_dir=$2 +ready_file=$3 + +until [ -f "$source_dir/index.php" ] && + [ -f "$source_dir/state-lock.php" ] && + cp "$source_dir/state-lock.php" "$target_dir/" && + cp "$source_dir/index.php" "$target_dir/index.php.new" && + mv "$target_dir/index.php.new" "$target_dir/index.php" +do + rm -f "$target_dir/index.php.new" + sleep 1 +done + +touch "$ready_file" diff --git a/.gitlab/test-request-replayer-concurrency.php b/.gitlab/test-request-replayer-concurrency.php new file mode 100644 index 0000000000..69decb838b --- /dev/null +++ b/.gitlab/test-request-replayer-concurrency.php @@ -0,0 +1,245 @@ + ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']], + $pipes, + dirname(__DIR__), + $environment + ); + if (!is_resource($process)) { + throw new RuntimeException('Failed to start: ' . implode(' ', $command)); + } + fclose($pipes[0]); + return [$process, $pipes]; +} + +function stopProcess(?array $processAndPipes): void +{ + if ($processAndPipes === null) { + return; + } + [$process, $pipes] = $processAndPipes; + if (proc_get_status($process)['running']) { + proc_terminate($process); + } + fclose($pipes[1]); + fclose($pipes[2]); + proc_close($process); +} + +function request(string $url, string $token = '', float $timeout = 2): string +{ + $headers = $token === '' ? '' : "X-Datadog-Test-Session-Token: $token"; + $response = @file_get_contents($url, false, stream_context_create([ + 'http' => ['header' => $headers, 'timeout' => $timeout], + ])); + if ($response === false) { + throw new RuntimeException("Request failed: $url"); + } + return $response; +} + +function openPostClient(int $port, string $token, string $body) +{ + $client = stream_socket_client("tcp://127.0.0.1:$port", $errorCode, $errorMessage, 1); + if ($client === false) { + throw new RuntimeException("Failed to connect client: $errorMessage ($errorCode)"); + } + $length = strlen($body); + fwrite( + $client, + "POST /v0.4/traces HTTP/1.1\r\nHost: 127.0.0.1\r\n" + . "X-Datadog-Test-Session-Token: $token\r\nContent-Type: application/json\r\n" + . "Content-Length: $length\r\nConnection: close\r\n\r\n$body" + ); + stream_set_timeout($client, 10); + return $client; +} + +$repoRoot = dirname(__DIR__); +$generatedYaml = shell_exec(escapeshellarg(PHP_BINARY) . ' ' . escapeshellarg($repoRoot . '/.gitlab/generate-tracer.php')); +if ($generatedYaml === null) { + throw new RuntimeException('Failed to generate the tracer pipeline'); +} +$workerCount = preg_match('/^\s+PHP_CLI_SERVER_WORKERS:\s*["\']?(\d+)["\']?\s*$/m', $generatedYaml, $matches) + ? (int) $matches[1] + : 1; +if ($workerCount <= 12) { + throw new RuntimeException("Request-replayer needs more than 12 workers, got $workerCount"); +} +foreach ([ + 'install-request-replayer-source.sh', + 'request-replayer-router.php', + 'request-replayer-ready', + 'REQUEST_REPLAYER_METRICS_SERVER_MANAGED: "1"', + 'while true; do php /var/www/metricsserver.php; sleep 1; done', + 'echo "$!" > /tmp/metrics-server.pid', +] as $expected) { + if (!str_contains($generatedYaml, $expected)) { + throw new RuntimeException("Generated request-replayer service is missing: $expected"); + } +} + +$server = null; +$installer = null; +$clients = []; +$lock = null; +$temporaryPaths = []; + +try { + $stateDirectory = sys_get_temp_dir() . '/request-replayer-state-' . getmypid(); + if (!mkdir($stateDirectory)) { + throw new RuntimeException("Failed to create $stateDirectory"); + } + $temporaryPaths[] = $stateDirectory; + + $installSource = $stateDirectory . '/install-source'; + $installTarget = $stateDirectory . '/install-target'; + $installReady = $stateDirectory . '/install-ready'; + mkdir($installSource); + $installer = startProcess([ + 'sh', + $repoRoot . '/.gitlab/install-request-replayer-source.sh', + $installSource, + $installTarget, + $installReady, + ], getenv()); + file_put_contents($installSource . '/state-lock.php', 'state lock'); + usleep(200_000); + if (file_exists($installReady)) { + throw new RuntimeException('Installer completed without index.php or a writable destination'); + } + file_put_contents($installSource . '/index.php', 'router'); + usleep(1_100_000); + if (file_exists($installReady)) { + throw new RuntimeException('Installer did not remain fail-closed after a copy failure'); + } + mkdir($installTarget); + $deadline = microtime(true) + 3; + while (!file_exists($installReady) && microtime(true) < $deadline) { + usleep(20_000); + } + if (!file_exists($installReady) + || file_get_contents($installTarget . '/index.php') !== 'router' + || file_get_contents($installTarget . '/state-lock.php') !== 'state lock' + ) { + throw new RuntimeException('Installer did not recover and install both source files'); + } + stopProcess($installer); + $installer = null; + + $docroot = $stateDirectory . '/docroot'; + if (!mkdir($docroot . '/vendor', 0777, true)) { + throw new RuntimeException("Failed to create $docroot"); + } + copy($repoRoot . '/dockerfiles/services/request-replayer/src/index.php', $docroot . '/index.php'); + copy($repoRoot . '/dockerfiles/services/request-replayer/src/state-lock.php', $docroot . '/state-lock.php'); + file_put_contents($docroot . '/vendor/autoload.php', " $id])); + } + foreach ($clients as $client) { + stream_get_contents($client); + fclose($client); + } + $clients = []; + + $requests = json_decode(request("$baseUrl/replay", 'shared'), true, flags: JSON_THROW_ON_ERROR); + $ids = array_map(static function (array $request): int { + return (int) json_decode($request['body'], true, flags: JSON_THROW_ON_ERROR)['id']; + }, $requests); + sort($ids, SORT_NUMERIC); + if ($ids !== range(0, 11)) { + throw new RuntimeException('Concurrent same-session requests were lost'); + } + + $slowStateDirectory = $stateDirectory . '/token-slow'; + if (!mkdir($slowStateDirectory)) { + throw new RuntimeException("Failed to create $slowStateDirectory"); + } + $lock = fopen($slowStateDirectory . '/.state.lock', 'c'); + if ($lock === false || !flock($lock, LOCK_EX)) { + throw new RuntimeException('Failed to hold the slow-session state lock'); + } + $slowClient = openPostClient($port, 'slow', '{}'); + $clients[] = $slowClient; + usleep(200_000); + $read = [$slowClient]; + $write = null; + $except = null; + if (stream_select($read, $write, $except, 0, 0) !== 0) { + throw new RuntimeException('Production request did not block on its session lock'); + } + if (request("$baseUrl/replay", 'other', 1) !== '') { + throw new RuntimeException('Unexpected replay data for the other session'); + } + fclose($lock); + $lock = null; + stream_get_contents($slowClient); + fclose($slowClient); + $clients = []; + + echo "request-replayer production locking passed with $workerCount workers\n"; +} finally { + stopProcess($installer); + if (is_resource($lock)) { + fclose($lock); + } + foreach ($clients as $client) { + fclose($client); + } + stopProcess($server); + foreach (array_reverse($temporaryPaths) as $path) { + if (!is_dir($path)) { + continue; + } + $iterator = new RecursiveIteratorIterator( + new RecursiveDirectoryIterator($path, FilesystemIterator::SKIP_DOTS), + RecursiveIteratorIterator::CHILD_FIRST + ); + foreach ($iterator as $child) { + $child->isDir() ? rmdir($child->getPathname()) : unlink($child->getPathname()); + } + rmdir($path); + } +} diff --git a/dockerfiles/services/request-replayer/src/index.php b/dockerfiles/services/request-replayer/src/index.php index ff7de4a3ad..8150341d37 100644 --- a/dockerfiles/services/request-replayer/src/index.php +++ b/dockerfiles/services/request-replayer/src/index.php @@ -3,6 +3,7 @@ error_reporting(\E_ALL); include __DIR__ . '/vendor/autoload.php'; +include __DIR__ . '/state-lock.php'; use MessagePack\BufferUnpacker; use MessagePack\UnpackOptions; @@ -51,9 +52,11 @@ function decodeDogStatsDMetrics($metrics) $temp_location = sys_get_temp_dir(); -$metricsServerPid = "$temp_location/metrics-server.pid"; -if (!file_exists($metricsServerPid)) { - shell_exec("nohup bash -c 'php metricsserver.php & pid=$!; echo \$pid > $metricsServerPid; wait \$pid; rm $metricsServerPid' > /dev/null 2>&1 &"); +if (!getenv('REQUEST_REPLAYER_METRICS_SERVER_MANAGED')) { + $metricsServerPid = "$temp_location/metrics-server.pid"; + if (!file_exists($metricsServerPid)) { + shell_exec("nohup bash -c 'php metricsserver.php & pid=$!; echo \$pid > $metricsServerPid; wait \$pid; rm $metricsServerPid' > /dev/null 2>&1 &"); + } } $token = $_SERVER["HTTP_X_DATADOG_TEST_SESSION_TOKEN"] ?? ""; @@ -81,6 +84,7 @@ function decodeDogStatsDMetrics($metrics) define('REQUEST_METRICS_LOG_FILE', getenv('REQUEST_METRICS_LOG_FILE') ?: ("$temp_location/metrics-log.txt")); define('REQUEST_STATS_FILE', getenv('REQUEST_STATS_FILE') ?: ("$temp_location/stats.json")); define('REQUEST_AGENT_INFO_FILE', getenv('REQUEST_AGENT_INFO_FILE') ?: ("$temp_location/agent-info.txt")); +define('REQUEST_STATE_LOCK_FILE', "$temp_location/.state.lock"); function logRequest($message, $data = '') { @@ -101,141 +105,153 @@ function logRequest($message, $data = '') trigger_error($message, $number); }); -$rc_configs = file_exists(REQUEST_RC_CONFIGS_FILE) ? json_decode(file_get_contents(REQUEST_RC_CONFIGS_FILE), true) : []; - switch ($uri) { case '/replay': - if (!file_exists(REQUEST_LATEST_DUMP_FILE)) { - logRequest('Cannot replay last request; request log does not exist'); - break; - } - $request = file_get_contents(REQUEST_LATEST_DUMP_FILE); - echo $request; - unlink(REQUEST_LATEST_DUMP_FILE); - unlink(REQUEST_LOG_FILE); - logRequest('Returned last request and deleted request log', $request); + $request = withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () { + if (!file_exists(REQUEST_LATEST_DUMP_FILE)) { + logRequest('Cannot replay last request; request log does not exist'); + return null; + } + $request = file_get_contents(REQUEST_LATEST_DUMP_FILE); + unlink(REQUEST_LATEST_DUMP_FILE); + unlink(REQUEST_LOG_FILE); + logRequest('Returned last request and deleted request log', $request); + return $request; + }); + echo $request ?? ''; break; case '/replay-metrics': - if (!file_exists(REQUEST_METRICS_FILE)) { - logRequest('Cannot replay last request; metrics log does not exist'); - break; - } - $request = file_get_contents(REQUEST_METRICS_FILE); - echo $request; - unlink(REQUEST_METRICS_FILE); - unlink(REQUEST_METRICS_LOG_FILE); - logRequest('Returned last metrics and deleted metrics log', $request); + $request = withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () { + if (!file_exists(REQUEST_METRICS_FILE)) { + logRequest('Cannot replay last request; metrics log does not exist'); + return null; + } + $request = file_get_contents(REQUEST_METRICS_FILE); + unlink(REQUEST_METRICS_FILE); + unlink(REQUEST_METRICS_LOG_FILE); + logRequest('Returned last metrics and deleted metrics log', $request); + return $request; + }); + echo $request ?? ''; break; case '/replay-stats': - if (!file_exists(REQUEST_STATS_FILE)) { - logRequest('Cannot replay stats; stats log does not exist'); - break; - } - $request = file_get_contents(REQUEST_STATS_FILE); - echo $request; - unlink(REQUEST_STATS_FILE); - logRequest('Returned stats and deleted stats log', $request); + $request = withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () { + if (!file_exists(REQUEST_STATS_FILE)) { + logRequest('Cannot replay stats; stats log does not exist'); + return null; + } + $request = file_get_contents(REQUEST_STATS_FILE); + unlink(REQUEST_STATS_FILE); + logRequest('Returned stats and deleted stats log', $request); + return $request; + }); + echo $request ?? ''; break; case '/replay-rc-requests': - if (!file_exists(REQUEST_RC_REQUESTS_FILE)) { - logRequest('Cannot replay RC requests; RC requests log does not exist'); - break; - } - $request = file_get_contents(REQUEST_RC_REQUESTS_FILE); - echo $request; - unlink(REQUEST_RC_REQUESTS_FILE); - logRequest('Returned RC requests and deleted RC requests log', $request); + $request = withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () { + if (!file_exists(REQUEST_RC_REQUESTS_FILE)) { + logRequest('Cannot replay RC requests; RC requests log does not exist'); + return null; + } + $request = file_get_contents(REQUEST_RC_REQUESTS_FILE); + unlink(REQUEST_RC_REQUESTS_FILE); + logRequest('Returned RC requests and deleted RC requests log', $request); + return $request; + }); + echo $request ?? ''; break; case '/clear-dumped-data': - if (!file_exists(REQUEST_LATEST_DUMP_FILE) && !file_exists(REQUEST_METRICS_FILE) && !file_exists(REQUEST_RC_CONFIGS_FILE)) { - logRequest('Cannot delete request log; request log does not exist'); - break; - } - if (file_exists(REQUEST_RC_CONFIGS_FILE)) { - unlink(REQUEST_RC_CONFIGS_FILE); - } - if (file_exists(REQUEST_LATEST_DUMP_FILE)) { - unlink(REQUEST_LATEST_DUMP_FILE); - unlink(REQUEST_LOG_FILE); - } - if (file_exists(REQUEST_METRICS_FILE)) { - unlink(REQUEST_METRICS_FILE); - unlink(REQUEST_METRICS_LOG_FILE); - } - if (file_exists(REQUEST_STATS_FILE)) { - unlink(REQUEST_STATS_FILE); - } - if (file_exists(REQUEST_NEXT_RESPONSE_FILE)) { - unlink(REQUEST_NEXT_RESPONSE_FILE); - } - if (file_exists(REQUEST_AGENT_INFO_FILE)) { - unlink(REQUEST_AGENT_INFO_FILE); - } - if (file_exists(REQUEST_RC_REQUESTS_FILE)) { - unlink(REQUEST_RC_REQUESTS_FILE); - } - logRequest('Deleted request log'); + withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () { + if (!file_exists(REQUEST_LATEST_DUMP_FILE) && !file_exists(REQUEST_METRICS_FILE) && !file_exists(REQUEST_RC_CONFIGS_FILE)) { + logRequest('Cannot delete request log; request log does not exist'); + return; + } + foreach ([ + REQUEST_RC_CONFIGS_FILE, + REQUEST_LATEST_DUMP_FILE, + REQUEST_LOG_FILE, + REQUEST_METRICS_FILE, + REQUEST_METRICS_LOG_FILE, + REQUEST_STATS_FILE, + REQUEST_NEXT_RESPONSE_FILE, + REQUEST_AGENT_INFO_FILE, + REQUEST_RC_REQUESTS_FILE, + ] as $file) { + if (file_exists($file)) { + unlink($file); + } + } + logRequest('Deleted request log'); + }); break; case '/next-response': $raw = file_get_contents('php://input'); - file_put_contents(REQUEST_NEXT_RESPONSE_FILE, $raw); + withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () use ($raw) { + file_put_contents(REQUEST_NEXT_RESPONSE_FILE, $raw); + }); break; case '/add-rc-config-file': - $rc_configs[$_GET["path"]] = ["service" => $_GET["service"], "data" => file_get_contents('php://input')]; - file_put_contents(REQUEST_RC_CONFIGS_FILE, json_encode($rc_configs, JSON_UNESCAPED_SLASHES)); + $raw = file_get_contents('php://input'); + withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () use ($raw) { + $rc_configs = file_exists(REQUEST_RC_CONFIGS_FILE) ? json_decode(file_get_contents(REQUEST_RC_CONFIGS_FILE), true) : []; + $rc_configs[$_GET["path"]] = ["service" => $_GET["service"], "data" => $raw]; + file_put_contents(REQUEST_RC_CONFIGS_FILE, json_encode($rc_configs, JSON_UNESCAPED_SLASHES)); + }); break; case '/del-rc-config-file': - unset($rc_configs[$_GET["path"]]); - file_put_contents(REQUEST_RC_CONFIGS_FILE, json_encode($rc_configs, JSON_UNESCAPED_SLASHES)); + withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () { + $rc_configs = file_exists(REQUEST_RC_CONFIGS_FILE) ? json_decode(file_get_contents(REQUEST_RC_CONFIGS_FILE), true) : []; + unset($rc_configs[$_GET["path"]]); + file_put_contents(REQUEST_RC_CONFIGS_FILE, json_encode($rc_configs, JSON_UNESCAPED_SLASHES)); + }); break; case '/v0.7/config': $request = file_get_contents('php://input'); logRequest("Requested remote config", $request); - - if (file_exists(REQUEST_RC_REQUESTS_FILE)) { - $tracesStack = json_decode(file_get_contents(REQUEST_RC_REQUESTS_FILE), true); - } else { - $tracesStack = []; - } - $tracesStack[] = ['uri' => $_SERVER['REQUEST_URI'], 'headers' => getallheaders(), 'body' => $request]; - file_put_contents(REQUEST_RC_REQUESTS_FILE, json_encode($tracesStack)); - - $request = json_decode($request, true); - $recentUpdate = @filemtime(REQUEST_RC_CONFIGS_FILE) > time() - 2; - $response = [ - "roots" => [], - "targets" => [ - "signatures" => [], - "signed" => [ - "_type" => "targets", - "custom" => [ - "opaque_backend_state" => "foobarbaz", - "agent_refresh_interval" => ($recentUpdate ? 10 : 10000) * 1000000, // in ns + $response = withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () use ($request) { + $tracesStack = file_exists(REQUEST_RC_REQUESTS_FILE) ? json_decode(file_get_contents(REQUEST_RC_REQUESTS_FILE), true) : []; + $tracesStack[] = ['uri' => $_SERVER['REQUEST_URI'], 'headers' => getallheaders(), 'body' => $request]; + file_put_contents(REQUEST_RC_REQUESTS_FILE, json_encode($tracesStack)); + + $decodedRequest = json_decode($request, true); + $rc_configs = file_exists(REQUEST_RC_CONFIGS_FILE) ? json_decode(file_get_contents(REQUEST_RC_CONFIGS_FILE), true) : []; + $recentUpdate = @filemtime(REQUEST_RC_CONFIGS_FILE) > time() - 2; + $response = [ + "roots" => [], + "targets" => [ + "signatures" => [], + "signed" => [ + "_type" => "targets", + "custom" => [ + "opaque_backend_state" => "foobarbaz", + "agent_refresh_interval" => ($recentUpdate ? 10 : 10000) * 1000000, // in ns + ], + "expires" => "9999-12-31T23:59:59Z", + "spec_version" => "1.0.0", + "targets" => new \StdClass, + "version" => 1, ], - "expires" => "9999-12-31T23:59:59Z", - "spec_version" => "1.0.0", - "targets" => new \StdClass, - "version" => 1, ], - ], - "target_files" => [], - "client_configs" => [], - ]; - foreach ($rc_configs as $path => $config) { - if ($config["service"] == $request["client"]["client_tracer"]["service"]) { - $content = $config["data"]; - $response["targets"]["signed"]["targets"]->$path = [ - "custom" => ["v" => strlen($path)], - "hashes" => ["sha256" => hash("sha256", $content)], - "length" => strlen($content), - ]; - $response["target_files"][] = [ - "path" => $path, - "raw" => base64_encode($content), - ]; - $response["client_configs"][] = $path; + "target_files" => [], + "client_configs" => [], + ]; + foreach ($rc_configs as $path => $config) { + if ($config["service"] == $decodedRequest["client"]["client_tracer"]["service"]) { + $content = $config["data"]; + $response["targets"]["signed"]["targets"]->$path = [ + "custom" => ["v" => strlen($path)], + "hashes" => ["sha256" => hash("sha256", $content)], + "length" => strlen($content), + ]; + $response["target_files"][] = [ + "path" => $path, + "raw" => base64_encode($content), + ]; + $response["client_configs"][] = $path; + } } - } + return $response; + }); logRequest("Returned remote config", json_encode($response, JSON_UNESCAPED_SLASHES)); $response["targets"] = base64_encode(json_encode($response["targets"], JSON_UNESCAPED_SLASHES)); echo json_encode($response, JSON_UNESCAPED_SLASHES); @@ -243,24 +259,25 @@ function logRequest($message, $data = '') case "/metrics": $_SERVER['REQUEST_URI'] = $uri; logRequest('Logged new metrics', json_encode($decodedMetrics)); - foreach ($decodedMetrics as $metric) { - file_put_contents(REQUEST_METRICS_LOG_FILE, json_encode($metric) . "\n", FILE_APPEND); - - if (file_exists(REQUEST_METRICS_FILE)) { - $allMetrics = json_decode(file_get_contents(REQUEST_METRICS_FILE), true); - } else { - $allMetrics = []; + withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () use ($decodedMetrics) { + $allMetrics = file_exists(REQUEST_METRICS_FILE) ? json_decode(file_get_contents(REQUEST_METRICS_FILE), true) : []; + foreach ($decodedMetrics as $metric) { + file_put_contents(REQUEST_METRICS_LOG_FILE, json_encode($metric) . "\n", FILE_APPEND); + $allMetrics[] = $metric; } - $allMetrics[] = $metric; file_put_contents(REQUEST_METRICS_FILE, json_encode($allMetrics)); - } + }); break; case '/set-agent-info': $raw = file_get_contents('php://input'); - file_put_contents(REQUEST_AGENT_INFO_FILE, $raw); + withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () use ($raw) { + file_put_contents(REQUEST_AGENT_INFO_FILE, $raw); + }); break; case '/info': - $file = @file_get_contents(REQUEST_AGENT_INFO_FILE) ?: "{}"; + $file = withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () { + return @file_get_contents(REQUEST_AGENT_INFO_FILE) ?: "{}"; + }); logRequest('Requested /info endpoint, returning ' . $file); header("datadog-agent-state: " . sha1($file)); echo $file; @@ -288,13 +305,11 @@ function logRequest($message, $data = '') 'headers' => getallheaders(), 'body' => $body, ]; - if (file_exists(REQUEST_STATS_FILE)) { - $statsStack = json_decode(file_get_contents(REQUEST_STATS_FILE), true); - } else { - $statsStack = []; - } - $statsStack[] = $newStatsRequest; - file_put_contents(REQUEST_STATS_FILE, json_encode($statsStack)); + withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () use ($newStatsRequest) { + $statsStack = file_exists(REQUEST_STATS_FILE) ? json_decode(file_get_contents(REQUEST_STATS_FILE), true) : []; + $statsStack[] = $newStatsRequest; + file_put_contents(REQUEST_STATS_FILE, json_encode($statsStack)); + }); logRequest('Logged stats request', $body); break; default: @@ -366,22 +381,21 @@ function logRequest($message, $data = '') $newIncomingRequest["body"] = $body; } - if (file_exists(REQUEST_LATEST_DUMP_FILE)) { - $tracesStack = json_decode(file_get_contents(REQUEST_LATEST_DUMP_FILE), true); - } else { - $tracesStack = []; - } - - $tracesStack[] = $newIncomingRequest; $newIncomingRequestJson = json_encode($newIncomingRequest); - - file_put_contents(REQUEST_LATEST_DUMP_FILE, json_encode($tracesStack)); - file_put_contents(REQUEST_LOG_FILE, $newIncomingRequestJson . "\n", FILE_APPEND); - logRequest('Logged new request', $newIncomingRequestJson); - - if (file_exists(REQUEST_NEXT_RESPONSE_FILE)) { - readfile(REQUEST_NEXT_RESPONSE_FILE); + $nextResponse = withRequestReplayerStateLock(REQUEST_STATE_LOCK_FILE, function () use ($newIncomingRequest, $newIncomingRequestJson) { + $tracesStack = file_exists(REQUEST_LATEST_DUMP_FILE) ? json_decode(file_get_contents(REQUEST_LATEST_DUMP_FILE), true) : []; + $tracesStack[] = $newIncomingRequest; + file_put_contents(REQUEST_LATEST_DUMP_FILE, json_encode($tracesStack)); + file_put_contents(REQUEST_LOG_FILE, $newIncomingRequestJson . "\n", FILE_APPEND); + logRequest('Logged new request', $newIncomingRequestJson); + + if (!file_exists(REQUEST_NEXT_RESPONSE_FILE)) { + return null; + } + $response = file_get_contents(REQUEST_NEXT_RESPONSE_FILE); unlink(REQUEST_NEXT_RESPONSE_FILE); - } + return $response; + }); + echo $nextResponse ?? ''; break; } diff --git a/dockerfiles/services/request-replayer/src/state-lock.php b/dockerfiles/services/request-replayer/src/state-lock.php new file mode 100644 index 0000000000..3cb2a55f0e --- /dev/null +++ b/dockerfiles/services/request-replayer/src/state-lock.php @@ -0,0 +1,19 @@ + Date: Thu, 13 Aug 2026 11:27:18 -0400 Subject: [PATCH 2/2] ci: avoid blocking request replayer test logs --- .gitlab/test-request-replayer-concurrency.php | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/.gitlab/test-request-replayer-concurrency.php b/.gitlab/test-request-replayer-concurrency.php index 69decb838b..c9541957a3 100644 --- a/.gitlab/test-request-replayer-concurrency.php +++ b/.gitlab/test-request-replayer-concurrency.php @@ -6,12 +6,12 @@ exit(0); } -function startProcess(array $command, array $environment): array +function startProcess(array $command, array $environment, string $outputFile) { $pipes = []; $process = proc_open( $command, - [0 => ['pipe', 'r'], 1 => ['pipe', 'w'], 2 => ['pipe', 'w']], + [0 => ['pipe', 'r'], 1 => ['file', $outputFile, 'a'], 2 => ['file', $outputFile, 'a']], $pipes, dirname(__DIR__), $environment @@ -20,20 +20,17 @@ function startProcess(array $command, array $environment): array throw new RuntimeException('Failed to start: ' . implode(' ', $command)); } fclose($pipes[0]); - return [$process, $pipes]; + return $process; } -function stopProcess(?array $processAndPipes): void +function stopProcess($process): void { - if ($processAndPipes === null) { + if ($process === null) { return; } - [$process, $pipes] = $processAndPipes; if (proc_get_status($process)['running']) { proc_terminate($process); } - fclose($pipes[1]); - fclose($pipes[2]); proc_close($process); } @@ -113,7 +110,7 @@ function openPostClient(int $port, string $token, string $body) $installSource, $installTarget, $installReady, - ], getenv()); + ], getenv(), $stateDirectory . '/installer.log'); file_put_contents($installSource . '/state-lock.php', 'state lock'); usleep(200_000); if (file_exists($installReady)) { @@ -159,7 +156,11 @@ function openPostClient(int $port, string $token, string $body) $environment['PHP_CLI_SERVER_WORKERS'] = (string) $workerCount; $environment['REQUEST_REPLAYER_METRICS_SERVER_MANAGED'] = '1'; $environment['TMPDIR'] = $stateDirectory; - $server = startProcess([PHP_BINARY, '-n', '-S', "127.0.0.1:$port", $docroot . '/index.php'], $environment); + $server = startProcess( + [PHP_BINARY, '-n', '-S', "127.0.0.1:$port", $docroot . '/index.php'], + $environment, + $stateDirectory . '/server.log' + ); $ready = false; for ($attempt = 0; $attempt < 100; $attempt++) { @@ -220,6 +221,14 @@ function openPostClient(int $port, string $token, string $body) $clients = []; echo "request-replayer production locking passed with $workerCount workers\n"; +} catch (Throwable $exception) { + foreach (['installer.log', 'server.log'] as $logName) { + $log = $stateDirectory . '/' . $logName; + if (file_exists($log) && filesize($log) > 0) { + fwrite(STDERR, "=== $logName ===\n" . file_get_contents($log)); + } + } + throw $exception; } finally { stopProcess($installer); if (is_resource($lock)) {