mirror of
https://github.com/OutlineFoundation/outline-server.git
synced 2026-05-13 05:52:04 +00:00
feat(server): add bandwidth stats to the server metrics (#1636)
This commit is contained in:
Sander Bruens
GitHub
parent
dbca08fce3
commit
76a23c6e81
2 changed files with 209 additions and 64 deletions
|
|
@ -41,6 +41,19 @@ describe('PrometheusManagerMetrics', () => {
|
|||
const managerMetrics = new PrometheusManagerMetrics(
|
||||
new QueryMapPrometheusClient(
|
||||
{
|
||||
'sum(increase(shadowsocks_data_bytes_per_location{dir=~"c<p|p>t"}[300s]))': {
|
||||
resultType: 'vector',
|
||||
result: [
|
||||
{
|
||||
metric: {
|
||||
location: 'US',
|
||||
asn: '49490',
|
||||
asorg: 'Test AS Org',
|
||||
},
|
||||
value: [1739284734, '1234'],
|
||||
},
|
||||
],
|
||||
},
|
||||
'sum(increase(shadowsocks_data_bytes_per_location{dir=~"c<p|p>t"}[0s])) by (location, asn, asorg)':
|
||||
{
|
||||
resultType: 'vector',
|
||||
|
|
@ -93,6 +106,22 @@ describe('PrometheusManagerMetrics', () => {
|
|||
},
|
||||
},
|
||||
{
|
||||
'sum(increase(shadowsocks_data_bytes_per_location{dir=~"c<p|p>t"}[300s]))': {
|
||||
resultType: 'matrix',
|
||||
result: [
|
||||
{
|
||||
metric: {
|
||||
location: 'US',
|
||||
asn: '49490',
|
||||
asorg: 'Test AS Org',
|
||||
},
|
||||
values: [
|
||||
[1738959398, '5678'],
|
||||
[1739284734, '1234'],
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
'sum(increase(shadowsocks_data_bytes{dir=~"c<p|p>t"}[300s])) by (access_key)': {
|
||||
resultType: 'matrix',
|
||||
result: [
|
||||
|
|
@ -102,7 +131,7 @@ describe('PrometheusManagerMetrics', () => {
|
|||
},
|
||||
values: [
|
||||
[1738959398, '1000'],
|
||||
[1738959398, '2000'],
|
||||
[1739284734, '2000'],
|
||||
],
|
||||
},
|
||||
],
|
||||
|
|
@ -116,7 +145,7 @@ describe('PrometheusManagerMetrics', () => {
|
|||
},
|
||||
values: [
|
||||
[1738959398, '1000'],
|
||||
[1738959398, '0'],
|
||||
[1739284734, '0'],
|
||||
],
|
||||
},
|
||||
],
|
||||
|
|
@ -128,19 +157,38 @@ describe('PrometheusManagerMetrics', () => {
|
|||
const serverMetrics = await managerMetrics.getServerMetrics({seconds: 0});
|
||||
|
||||
expect(JSON.stringify(serverMetrics, null, 2)).toEqual(`{
|
||||
"server": [
|
||||
{
|
||||
"location": "US",
|
||||
"asn": 49490,
|
||||
"asOrg": "Test AS Org",
|
||||
"dataTransferred": {
|
||||
"server": {
|
||||
"tunnelTime": {
|
||||
"seconds": 1000
|
||||
},
|
||||
"dataTransferred": {
|
||||
"total": {
|
||||
"bytes": 1000
|
||||
},
|
||||
"tunnelTime": {
|
||||
"seconds": 1000
|
||||
"current": {
|
||||
"bytes": 1234
|
||||
},
|
||||
"peak": {
|
||||
"data": {
|
||||
"bytes": 5678
|
||||
},
|
||||
"timestamp": 1738959398
|
||||
}
|
||||
}
|
||||
],
|
||||
},
|
||||
"locations": [
|
||||
{
|
||||
"location": "US",
|
||||
"asn": 49490,
|
||||
"asOrg": "Test AS Org",
|
||||
"dataTransferred": {
|
||||
"bytes": 1000
|
||||
},
|
||||
"tunnelTime": {
|
||||
"seconds": 1000
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"accessKeys": [
|
||||
{
|
||||
"accessKeyId": 0,
|
||||
|
|
@ -152,9 +200,9 @@ describe('PrometheusManagerMetrics', () => {
|
|||
},
|
||||
"connection": {
|
||||
"lastConnected": 1738959398,
|
||||
"lastTrafficSeen": 1738959398,
|
||||
"peakDevices": {
|
||||
"count": 4,
|
||||
"lastTrafficSeen": 1739284734,
|
||||
"peakDeviceCount": {
|
||||
"data": 4,
|
||||
"timestamp": 1738959398
|
||||
}
|
||||
}
|
||||
|
|
@ -168,6 +216,19 @@ describe('PrometheusManagerMetrics', () => {
|
|||
const managerMetrics = new PrometheusManagerMetrics(
|
||||
new QueryMapPrometheusClient(
|
||||
{
|
||||
'sum(increase(shadowsocks_data_bytes_per_location{dir=~"c<p|p>t"}[300s]))': {
|
||||
resultType: 'vector',
|
||||
result: [
|
||||
{
|
||||
metric: {
|
||||
location: 'US',
|
||||
asn: '49490',
|
||||
asorg: 'Test AS Org',
|
||||
},
|
||||
value: [1739284734, '1234'],
|
||||
},
|
||||
],
|
||||
},
|
||||
'sum(increase(shadowsocks_data_bytes_per_location{dir=~"c<p|p>t"}[0s])) by (location, asn, asorg)':
|
||||
{
|
||||
resultType: 'vector',
|
||||
|
|
@ -218,6 +279,22 @@ describe('PrometheusManagerMetrics', () => {
|
|||
},
|
||||
},
|
||||
{
|
||||
'sum(increase(shadowsocks_data_bytes_per_location{dir=~"c<p|p>t"}[300s]))': {
|
||||
resultType: 'matrix',
|
||||
result: [
|
||||
{
|
||||
metric: {
|
||||
location: 'US',
|
||||
asn: '49490',
|
||||
asorg: 'Test AS Org',
|
||||
},
|
||||
values: [
|
||||
[1738959398, '5678'],
|
||||
[1739284734, '1234'],
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
'sum(increase(shadowsocks_data_bytes{dir=~"c<p|p>t"}[300s])) by (access_key)': {
|
||||
resultType: 'matrix',
|
||||
result: [
|
||||
|
|
@ -253,30 +330,49 @@ describe('PrometheusManagerMetrics', () => {
|
|||
const serverMetrics = await managerMetrics.getServerMetrics({seconds: 0});
|
||||
|
||||
expect(JSON.stringify(serverMetrics, null, 2)).toEqual(`{
|
||||
"server": [
|
||||
{
|
||||
"location": "CA",
|
||||
"asn": null,
|
||||
"asOrg": null,
|
||||
"dataTransferred": {
|
||||
"bytes": 0
|
||||
},
|
||||
"tunnelTime": {
|
||||
"seconds": 1000
|
||||
}
|
||||
"server": {
|
||||
"tunnelTime": {
|
||||
"seconds": 1000
|
||||
},
|
||||
{
|
||||
"location": "US",
|
||||
"asn": 49490,
|
||||
"asOrg": "Test AS Org",
|
||||
"dataTransferred": {
|
||||
"dataTransferred": {
|
||||
"total": {
|
||||
"bytes": 1000
|
||||
},
|
||||
"tunnelTime": {
|
||||
"seconds": 0
|
||||
"current": {
|
||||
"bytes": 1234
|
||||
},
|
||||
"peak": {
|
||||
"data": {
|
||||
"bytes": 5678
|
||||
},
|
||||
"timestamp": 1738959398
|
||||
}
|
||||
}
|
||||
],
|
||||
},
|
||||
"locations": [
|
||||
{
|
||||
"location": "CA",
|
||||
"asn": null,
|
||||
"asOrg": null,
|
||||
"dataTransferred": {
|
||||
"bytes": 0
|
||||
},
|
||||
"tunnelTime": {
|
||||
"seconds": 1000
|
||||
}
|
||||
},
|
||||
{
|
||||
"location": "US",
|
||||
"asn": 49490,
|
||||
"asOrg": "Test AS Org",
|
||||
"dataTransferred": {
|
||||
"bytes": 1000
|
||||
},
|
||||
"tunnelTime": {
|
||||
"seconds": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"accessKeys": [
|
||||
{
|
||||
"accessKeyId": 1,
|
||||
|
|
@ -289,8 +385,8 @@ describe('PrometheusManagerMetrics', () => {
|
|||
"connection": {
|
||||
"lastConnected": null,
|
||||
"lastTrafficSeen": null,
|
||||
"peakDevices": {
|
||||
"count": 0,
|
||||
"peakDeviceCount": {
|
||||
"data": 0,
|
||||
"timestamp": null
|
||||
}
|
||||
}
|
||||
|
|
@ -306,8 +402,8 @@ describe('PrometheusManagerMetrics', () => {
|
|||
"connection": {
|
||||
"lastConnected": 1738959398,
|
||||
"lastTrafficSeen": 1738959398,
|
||||
"peakDevices": {
|
||||
"count": 4,
|
||||
"peakDeviceCount": {
|
||||
"data": 4,
|
||||
"timestamp": 1738959398
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,18 +29,30 @@ interface Data {
|
|||
bytes: number;
|
||||
}
|
||||
|
||||
interface PeakDevices {
|
||||
count: number;
|
||||
interface TimedData<T> {
|
||||
data: T;
|
||||
timestamp: number | null;
|
||||
}
|
||||
|
||||
interface ConnectionStats {
|
||||
lastConnected: number | null;
|
||||
lastTrafficSeen: number | null;
|
||||
peakDevices: PeakDevices;
|
||||
peakDeviceCount: TimedData<number>;
|
||||
}
|
||||
|
||||
interface BandwidthStats {
|
||||
total: Data;
|
||||
current: Data;
|
||||
peak: TimedData<Data>;
|
||||
}
|
||||
|
||||
interface ServerMetricsServerEntry {
|
||||
tunnelTime: Duration;
|
||||
dataTransferred: BandwidthStats;
|
||||
locations: ServerMetricsLocationEntry[];
|
||||
}
|
||||
|
||||
interface ServerMetricsLocationEntry {
|
||||
location: string;
|
||||
asn: number | null;
|
||||
asOrg: string | null;
|
||||
|
|
@ -56,7 +68,7 @@ interface ServerMetricsAccessKeyEntry {
|
|||
}
|
||||
|
||||
interface ServerMetrics {
|
||||
server: ServerMetricsServerEntry[];
|
||||
server: ServerMetricsServerEntry;
|
||||
accessKeys: ServerMetricsAccessKeyEntry[];
|
||||
}
|
||||
|
||||
|
|
@ -88,7 +100,7 @@ export class PrometheusManagerMetrics implements ManagerMetrics {
|
|||
}
|
||||
|
||||
async getServerMetrics(timeframe: Duration): Promise<ServerMetrics> {
|
||||
const now = new Date().getTime();
|
||||
const now = new Date().getTime() / 1000;
|
||||
// We need to calculate consistent start and end times for Prometheus range
|
||||
// queries. Rounding the end time *up* to the nearest multiple of the step
|
||||
// prevents time "drift" between queries, which is crucial for reliable step
|
||||
|
|
@ -97,11 +109,12 @@ export class PrometheusManagerMetrics implements ManagerMetrics {
|
|||
// windows are queried each time, leading to more stable and predictable
|
||||
// results.
|
||||
const end =
|
||||
Math.ceil(now / (PROMETHEUS_RANGE_QUERY_STEP_SECONDS * 1000)) *
|
||||
PROMETHEUS_RANGE_QUERY_STEP_SECONDS;
|
||||
Math.ceil(now / PROMETHEUS_RANGE_QUERY_STEP_SECONDS) * PROMETHEUS_RANGE_QUERY_STEP_SECONDS;
|
||||
const start = end - timeframe.seconds;
|
||||
|
||||
const [
|
||||
totalDataTransferred,
|
||||
totalDataTransferredRange,
|
||||
dataTransferredByLocation,
|
||||
tunnelTimeByLocation,
|
||||
dataTransferredByAccessKey,
|
||||
|
|
@ -109,6 +122,15 @@ export class PrometheusManagerMetrics implements ManagerMetrics {
|
|||
dataTransferredByAccessKeyRange,
|
||||
tunnelTimeByAccessKeyRange,
|
||||
] = await Promise.all([
|
||||
this.prometheusClient.query(
|
||||
`sum(increase(shadowsocks_data_bytes_per_location{dir=~"c<p|p>t"}[${PROMETHEUS_RANGE_QUERY_STEP_SECONDS}s]))`
|
||||
),
|
||||
this.prometheusClient.queryRange(
|
||||
`sum(increase(shadowsocks_data_bytes_per_location{dir=~"c<p|p>t"}[${PROMETHEUS_RANGE_QUERY_STEP_SECONDS}s]))`,
|
||||
start,
|
||||
end,
|
||||
`${PROMETHEUS_RANGE_QUERY_STEP_SECONDS}s`
|
||||
),
|
||||
this.prometheusClient.query(
|
||||
`sum(increase(shadowsocks_data_bytes_per_location{dir=~"c<p|p>t"}[${timeframe.seconds}s])) by (location, asn, asorg)`
|
||||
),
|
||||
|
|
@ -135,28 +157,56 @@ export class PrometheusManagerMetrics implements ManagerMetrics {
|
|||
),
|
||||
]);
|
||||
|
||||
const serverMap = new Map<string, ServerMetricsServerEntry>();
|
||||
for (const result of tunnelTimeByLocation.result) {
|
||||
const entry = getServerMetricsServerEntry(serverMap, result.metric);
|
||||
entry.tunnelTime.seconds = result.value ? parseFloat(result.value[1]) : 0;
|
||||
const serverMetrics: ServerMetricsServerEntry = {
|
||||
tunnelTime: {seconds: 0},
|
||||
dataTransferred: {
|
||||
total: {bytes: 0},
|
||||
current: {bytes: 0},
|
||||
peak: {data: {bytes: 0}, timestamp: null},
|
||||
},
|
||||
locations: [],
|
||||
};
|
||||
for (const result of totalDataTransferred.result) {
|
||||
const bytes = result.value ? parseFloat(result.value[1]) : 0;
|
||||
serverMetrics.dataTransferred.current.bytes = bytes;
|
||||
break; // There should only be one result.
|
||||
}
|
||||
for (const result of totalDataTransferredRange.result) {
|
||||
const peakDataTransferred = findPeak(result.values ?? []);
|
||||
if (peakDataTransferred !== null) {
|
||||
const peakValue = parseFloat(peakDataTransferred[1]);
|
||||
if (peakValue > 0) {
|
||||
serverMetrics.dataTransferred.peak.data.bytes = peakValue;
|
||||
serverMetrics.dataTransferred.peak.timestamp = Math.min(now, peakDataTransferred[0]);
|
||||
}
|
||||
}
|
||||
break; // There should only be one result.
|
||||
}
|
||||
|
||||
for (const result of dataTransferredByLocation.result) {
|
||||
const entry = getServerMetricsServerEntry(serverMap, result.metric);
|
||||
entry.dataTransferred.bytes = result.value ? parseFloat(result.value[1]) : 0;
|
||||
const locationMap = new Map<string, ServerMetricsLocationEntry>();
|
||||
for (const result of tunnelTimeByLocation.result) {
|
||||
const entry = getServerMetricsLocationEntry(locationMap, result.metric);
|
||||
const tunnelTime = result.value ? parseFloat(result.value[1]) : 0;
|
||||
entry.tunnelTime.seconds = tunnelTime;
|
||||
serverMetrics.tunnelTime.seconds += tunnelTime;
|
||||
}
|
||||
for (const result of dataTransferredByLocation.result) {
|
||||
const entry = getServerMetricsLocationEntry(locationMap, result.metric);
|
||||
const bytes = result.value ? parseFloat(result.value[1]) : 0;
|
||||
entry.dataTransferred.bytes = bytes;
|
||||
serverMetrics.dataTransferred.total.bytes += bytes;
|
||||
}
|
||||
serverMetrics.locations = Array.from(locationMap.values());
|
||||
|
||||
const accessKeyMap = new Map<string, ServerMetricsAccessKeyEntry>();
|
||||
for (const result of tunnelTimeByAccessKey.result) {
|
||||
const entry = getServerMetricsAccessKeyEntry(accessKeyMap, result.metric);
|
||||
entry.tunnelTime.seconds = result.value ? parseFloat(result.value[1]) : 0;
|
||||
}
|
||||
|
||||
for (const result of dataTransferredByAccessKey.result) {
|
||||
const entry = getServerMetricsAccessKeyEntry(accessKeyMap, result.metric);
|
||||
entry.dataTransferred.bytes = result.value ? parseFloat(result.value[1]) : 0;
|
||||
}
|
||||
|
||||
for (const result of tunnelTimeByAccessKeyRange.result) {
|
||||
const entry = getServerMetricsAccessKeyEntry(accessKeyMap, result.metric);
|
||||
const lastConnected = findLastNonZero(result.values ?? []);
|
||||
|
|
@ -166,12 +216,11 @@ export class PrometheusManagerMetrics implements ManagerMetrics {
|
|||
const peakValue = parseFloat(peakTunnelTimeSec[1]);
|
||||
if (peakValue > 0) {
|
||||
const peakTunnelTimeOverTime = peakValue / PROMETHEUS_RANGE_QUERY_STEP_SECONDS;
|
||||
entry.connection.peakDevices.count = Math.ceil(peakTunnelTimeOverTime);
|
||||
entry.connection.peakDevices.timestamp = Math.min(now, peakTunnelTimeSec[0]);
|
||||
entry.connection.peakDeviceCount.data = Math.ceil(peakTunnelTimeOverTime);
|
||||
entry.connection.peakDeviceCount.timestamp = Math.min(now, peakTunnelTimeSec[0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const result of dataTransferredByAccessKeyRange.result) {
|
||||
const entry = getServerMetricsAccessKeyEntry(accessKeyMap, result.metric);
|
||||
const lastTrafficSeen = findLastNonZero(result.values ?? []);
|
||||
|
|
@ -179,16 +228,16 @@ export class PrometheusManagerMetrics implements ManagerMetrics {
|
|||
}
|
||||
|
||||
return {
|
||||
server: Array.from(serverMap.values()),
|
||||
server: serverMetrics,
|
||||
accessKeys: Array.from(accessKeyMap.values()),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function getServerMetricsServerEntry(
|
||||
map: Map<string, ServerMetricsServerEntry>,
|
||||
function getServerMetricsLocationEntry(
|
||||
map: Map<string, ServerMetricsLocationEntry>,
|
||||
metric: PrometheusMetric
|
||||
): ServerMetricsServerEntry {
|
||||
): ServerMetricsLocationEntry {
|
||||
const {location, asn, asorg} = metric;
|
||||
const key = `${location},${asn},${asorg}`;
|
||||
let entry = map.get(key);
|
||||
|
|
@ -219,8 +268,8 @@ function getServerMetricsAccessKeyEntry(
|
|||
connection: {
|
||||
lastConnected: null,
|
||||
lastTrafficSeen: null,
|
||||
peakDevices: {
|
||||
count: 0,
|
||||
peakDeviceCount: {
|
||||
data: 0,
|
||||
timestamp: null,
|
||||
},
|
||||
},
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue