From 257776530fc68b6c9dfe578e038231e852ea55ef Mon Sep 17 00:00:00 2001 From: Oskar Eichler Date: Thu, 27 Aug 2026 21:44:56 +0300 Subject: [PATCH 1/2] fix(server): bound Prometheus requests and handle response failures --- .../infrastructure/prometheus_scraper.ts | 70 +++++++++---------- 1 file changed, 32 insertions(+), 38 deletions(-) diff --git a/src/shadowbox/infrastructure/prometheus_scraper.ts b/src/shadowbox/infrastructure/prometheus_scraper.ts index af9ebad5..7c78c4ab 100644 --- a/src/shadowbox/infrastructure/prometheus_scraper.ts +++ b/src/shadowbox/infrastructure/prometheus_scraper.ts @@ -16,7 +16,7 @@ import * as child_process from 'child_process'; import * as fs from 'fs'; import * as http from 'http'; import * as jsyaml from 'js-yaml'; -import * as mkdirp from 'mkdirp'; +import fetch from 'node-fetch'; import * as path from 'path'; import * as logging from '../infrastructure/logging'; @@ -105,35 +105,32 @@ export class ApiPrometheusClient implements PrometheusClient { private readonly agent: http.Agent; constructor(private address: string) { - this.agent = new http.Agent({ keepAlive: true }); + this.agent = new http.Agent({keepAlive: true}); } - private request(url: string): Promise { - return new Promise((fulfill, reject) => { - const options = {agent: this.agent}; - http - .get(url, options, (response) => { - if (response.statusCode < 200 || response.statusCode > 299) { - reject(new Error(`Got error ${response.statusCode}`)); - response.resume(); - return; - } - let body = ''; - response.on('data', (data) => { - body += data; - }); - response.on('end', () => { - const result = JSON.parse(body) as QueryResult; - if (result.status !== 'success') { - return reject(new Error(`Error ${result.errorType}: ${result.error}`)); - } - fulfill(result.data); - }); - }) - .on('error', (e) => { - reject(new Error(`Failed to query prometheus API: ${e}`)); - }); - }); + private async request(url: string): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 30000); + try { + const response = await fetch(url, { + agent: this.agent, + signal: controller.signal, + redirect: 'error', + }); + if (!response.ok) { + throw new Error(`Got error ${response.status}`); + } + // Parsing and body-stream errors must reject, not escape an HTTP callback. + const result = (await response.json()) as QueryResult; + if (result.status !== 'success') { + throw new Error(`Error ${result.errorType}: ${result.error}`); + } + return result.data; + } finally { + clearTimeout(timeout); + // Also release the connection when status checks or body parsing fail. + controller.abort(); + } } query(query: string): Promise { @@ -161,18 +158,10 @@ export async function startPrometheus( } async function writePrometheusConfigToDisk(configFilename: string, configJson: {}) { - await mkdirp.sync(path.dirname(configFilename)); + await fs.promises.mkdir(path.dirname(configFilename), {recursive: true}); const ymlTxt = jsyaml.safeDump(configJson, {sortKeys: true}); // Write the file asynchronously to prevent blocking the node thread. - await new Promise((resolve, reject) => { - fs.writeFile(configFilename, ymlTxt, 'utf-8', (err) => { - if (err) { - reject(err); - } else { - resolve(); - } - }); - }); + await fs.promises.writeFile(configFilename, ymlTxt, 'utf-8'); } async function spawnPrometheusSubprocess( @@ -209,8 +198,13 @@ function isHttpEndpointHealthy(endpoint: string): Promise { return new Promise((resolve, _) => { http .get(endpoint, (response) => { + response.resume(); resolve(response.statusCode >= 200 && response.statusCode < 300); }) + .setTimeout(1000, function () { + this.destroy(); + resolve(false); + }) .on('error', () => { // Prometheus is not ready yet. resolve(false); From 617aaed30767b0a33ad5209912db30d8e25580dc Mon Sep 17 00:00:00 2001 From: Oskar Eichler Date: Thu, 27 Aug 2026 22:36:00 +0300 Subject: [PATCH 2/2] fix(server): bound readiness probes and reuse one recovery loop --- .../infrastructure/prometheus_scraper.ts | 53 ++++++++++++------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/src/shadowbox/infrastructure/prometheus_scraper.ts b/src/shadowbox/infrastructure/prometheus_scraper.ts index 7c78c4ab..54948274 100644 --- a/src/shadowbox/infrastructure/prometheus_scraper.ts +++ b/src/shadowbox/infrastructure/prometheus_scraper.ts @@ -169,20 +169,32 @@ async function spawnPrometheusSubprocess( processArgs: string[], prometheusEndpoint: string ): Promise { - logging.info('======== Starting Prometheus ========'); - logging.info(`${binaryFilename} ${processArgs.map((a) => `"${a}"`).join(' ')}`); - const runProcess = child_process.spawn(binaryFilename, processArgs); - runProcess.on('error', (error) => { - logging.error(`Error spawning Prometheus: ${error}`); - }); - runProcess.on('exit', (code, signal) => { - logging.error(`Prometheus has exited with error. Code: ${code}, Signal: ${signal}`); - logging.error('Restarting Prometheus...'); - spawnPrometheusSubprocess(binaryFilename, processArgs, prometheusEndpoint); - }); - // TODO(fortuna): Consider saving the output and expose it through the manager service. - runProcess.stdout.pipe(process.stdout); - runProcess.stderr.pipe(process.stderr); + let runProcess: child_process.ChildProcess; + let restartDelayMs = 1000; + const start = () => { + logging.info('======== Starting Prometheus ========'); + logging.info(`${binaryFilename} ${processArgs.map((a) => `"${a}"`).join(' ')}`); + const startedAt = Date.now(); + runProcess = child_process.spawn(binaryFilename, processArgs); + runProcess.on('error', (error) => { + logging.error(`Error spawning Prometheus: ${error}`); + }); + runProcess.once('close', (code, signal) => { + if (Date.now() - startedAt >= 60000) { + restartDelayMs = 1000; + } + logging.error( + `Prometheus exited. Code: ${code}, Signal: ${signal}; restarting in ${restartDelayMs}ms` + ); + setTimeout(start, restartDelayMs); + restartDelayMs = Math.min(restartDelayMs * 2, 30000); + }); + runProcess.stdout.pipe(process.stdout); + runProcess.stderr.pipe(process.stderr); + }; + start(); + // Use one readiness loop across retries, rather than creating another loop + // (and an abandoned promise) every time an unready child exits. await waitForPrometheusReady(`${prometheusEndpoint}/api/v1/status/flags`); logging.info('Prometheus is ready!'); return runProcess; @@ -196,18 +208,21 @@ async function waitForPrometheusReady(prometheusEndpoint: string) { function isHttpEndpointHealthy(endpoint: string): Promise { return new Promise((resolve, _) => { - http + const request = http .get(endpoint, (response) => { response.resume(); resolve(response.statusCode >= 200 && response.statusCode < 300); }) - .setTimeout(1000, function () { - this.destroy(); - resolve(false); - }) .on('error', () => { // Prometheus is not ready yet. resolve(false); }); + // An inactivity timeout can be kept alive by a trickling response. Bound + // the entire probe instead, including connecting and draining its body. + const timeout = setTimeout(() => { + request.destroy(); + resolve(false); + }, 1000); + request.once('close', () => clearTimeout(timeout)); }); }