CERBERUS/outline-server
1
0
Fork 0
mirror of https://github.com/OutlineFoundation/outline-server.git synced 2026-10-21 12:00:57 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge 617aaed307 into de566bfc9f

Browse source
This commit is contained in:
Oskar Eichler 2026-08-27 22:36:56 +03:00 • committed by GitHub
commit 5a047eb3f1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 62 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

115
src/shadowbox/infrastructure/prometheus_scraper.ts
View file

@ -16,7 +16,7 @@ import * as child_process from 'child_process';
import * as fs from 'fs';
import * as http from 'http';
import * as jsyaml from 'js-yaml';
import * as mkdirp from 'mkdirp';
import fetch from 'node-fetch';
import * as path from 'path';
import * as logging from '../infrastructure/logging';
@ -105,35 +105,32 @@ export class ApiPrometheusClient implements PrometheusClient {
private readonly agent: http.Agent;
constructor(private address: string) {
this.agent = new http.Agent({ keepAlive: true });
this.agent = new http.Agent({keepAlive: true});
}
private request(url: string): Promise<QueryResultData> {
return new Promise<QueryResultData>((fulfill, reject) => {
const options = {agent: this.agent};
http
.get(url, options, (response) => {
if (response.statusCode < 200 || response.statusCode > 299) {
reject(new Error(`Got error ${response.statusCode}`));
response.resume();
return;
}
let body = '';
response.on('data', (data) => {
body += data;
});
response.on('end', () => {
const result = JSON.parse(body) as QueryResult;
if (result.status !== 'success') {
return reject(new Error(`Error ${result.errorType}: ${result.error}`));
}
fulfill(result.data);
});
})
.on('error', (e) => {
reject(new Error(`Failed to query prometheus API: ${e}`));
});
});
private async request(url: string): Promise<QueryResultData> {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 30000);
try {
const response = await fetch(url, {
agent: this.agent,
signal: controller.signal,
redirect: 'error',
});
if (!response.ok) {
throw new Error(`Got error ${response.status}`);
}
// Parsing and body-stream errors must reject, not escape an HTTP callback.
const result = (await response.json()) as QueryResult;
if (result.status !== 'success') {
throw new Error(`Error ${result.errorType}: ${result.error}`);
}
return result.data;
} finally {
clearTimeout(timeout);
// Also release the connection when status checks or body parsing fail.
controller.abort();
}
}
query(query: string): Promise<QueryResultData> {
@ -161,18 +158,10 @@ export async function startPrometheus(
}
async function writePrometheusConfigToDisk(configFilename: string, configJson: {}) {
await mkdirp.sync(path.dirname(configFilename));
await fs.promises.mkdir(path.dirname(configFilename), {recursive: true});
const ymlTxt = jsyaml.safeDump(configJson, {sortKeys: true});
// Write the file asynchronously to prevent blocking the node thread.
await new Promise<void>((resolve, reject) => {
fs.writeFile(configFilename, ymlTxt, 'utf-8', (err) => {
if (err) {
reject(err);
} else {
resolve();
}
});
});
await fs.promises.writeFile(configFilename, ymlTxt, 'utf-8');
}
async function spawnPrometheusSubprocess(
@ -180,20 +169,32 @@ async function spawnPrometheusSubprocess(
processArgs: string[],
prometheusEndpoint: string
): Promise<child_process.ChildProcess> {
logging.info('======== Starting Prometheus ========');
logging.info(`${binaryFilename} ${processArgs.map((a) => `"${a}"`).join(' ')}`);
const runProcess = child_process.spawn(binaryFilename, processArgs);
runProcess.on('error', (error) => {
logging.error(`Error spawning Prometheus: ${error}`);
});
runProcess.on('exit', (code, signal) => {
logging.error(`Prometheus has exited with error. Code: ${code}, Signal: ${signal}`);
logging.error('Restarting Prometheus...');
spawnPrometheusSubprocess(binaryFilename, processArgs, prometheusEndpoint);
});
// TODO(fortuna): Consider saving the output and expose it through the manager service.
runProcess.stdout.pipe(process.stdout);
runProcess.stderr.pipe(process.stderr);
let runProcess: child_process.ChildProcess;
let restartDelayMs = 1000;
const start = () => {
logging.info('======== Starting Prometheus ========');
logging.info(`${binaryFilename} ${processArgs.map((a) => `"${a}"`).join(' ')}`);
const startedAt = Date.now();
runProcess = child_process.spawn(binaryFilename, processArgs);
runProcess.on('error', (error) => {
logging.error(`Error spawning Prometheus: ${error}`);
});
runProcess.once('close', (code, signal) => {
if (Date.now() - startedAt >= 60000) {
restartDelayMs = 1000;
}
logging.error(
`Prometheus exited. Code: ${code}, Signal: ${signal}; restarting in ${restartDelayMs}ms`
);
setTimeout(start, restartDelayMs);
restartDelayMs = Math.min(restartDelayMs * 2, 30000);
});
runProcess.stdout.pipe(process.stdout);
runProcess.stderr.pipe(process.stderr);
};
start();
// Use one readiness loop across retries, rather than creating another loop
// (and an abandoned promise) every time an unready child exits.
await waitForPrometheusReady(`${prometheusEndpoint}/api/v1/status/flags`);
logging.info('Prometheus is ready!');
return runProcess;
@ -207,13 +208,21 @@ async function waitForPrometheusReady(prometheusEndpoint: string) {
function isHttpEndpointHealthy(endpoint: string): Promise<boolean> {
return new Promise((resolve, _) => {
http
const request = http
.get(endpoint, (response) => {
response.resume();
resolve(response.statusCode >= 200 && response.statusCode < 300);
})
.on('error', () => {
// Prometheus is not ready yet.
resolve(false);
});
// An inactivity timeout can be kept alive by a trickling response. Bound
// the entire probe instead, including connecting and draining its body.
const timeout = setTimeout(() => {
request.destroy();
resolve(false);
}, 1000);
request.once('close', () => clearTimeout(timeout));
});
}