feat(metrics_server): add support for ASN (#1542)

* feat(metrics_server): add support for ASN

* Change from array to number.

* Add more debug logs for invalid returns.

package-lock.json

@@ -5,6 +5,7 @@
   "packages": {
     "": {
       "name": "outline-server",
+      "hasInstallScript": true,
       "workspaces": [
         "src/*"
       ],

src/metrics_server/README.md

@@ -21,6 +21,7 @@ The metrics server supports two URL paths:
     endUtcMs: number,
     userReports: [{
         countries: string[],
+        asn: number,
         bytesTransferred: number,
         tunnelTimeSec: number,
     }]

src/metrics_server/connection_metrics.spec.ts

@@ -28,6 +28,7 @@ const VALID_USER_REPORT: HourlyUserConnectionMetricsReport = {
 
 const VALID_USER_REPORT2: HourlyUserConnectionMetricsReport = {
   countries: ['UK'],
+  asn: 54321,
   bytesTransferred: 456,
 };
 
@@ -90,6 +91,7 @@ describe('postConnectionMetrics', () => {
         bytesTransferred: VALID_USER_REPORT.bytesTransferred,
         tunnelTimeSec: VALID_USER_REPORT.tunnelTimeSec,
         countries: VALID_USER_REPORT.countries,
+        asn: undefined,
       },
       {
         serverId: VALID_REPORT.serverId,
@@ -98,6 +100,7 @@ describe('postConnectionMetrics', () => {
         bytesTransferred: VALID_USER_REPORT2.bytesTransferred,
         tunnelTimeSec: VALID_USER_REPORT2.tunnelTimeSec,
         countries: VALID_USER_REPORT2.countries,
+        asn: VALID_USER_REPORT2.asn!,
       },
       {
         serverId: VALID_REPORT.serverId,
@@ -106,6 +109,7 @@ describe('postConnectionMetrics', () => {
         bytesTransferred: LEGACY_PER_LOCATION_USER_REPORT.bytesTransferred,
         tunnelTimeSec: LEGACY_PER_LOCATION_USER_REPORT.tunnelTimeSec,
         countries: LEGACY_PER_LOCATION_USER_REPORT.countries,
+        asn: undefined,
       },
     ];
     expect(table.rows).toEqual(rows);
@@ -215,11 +219,16 @@ describe('isValidConnectionMetricsReport', () => {
     report.userReports[0].countries = 'US' as unknown as string[];
     expect(isValidConnectionMetricsReport(report)).toBeFalse();
   });
-  it('returns false for `countries` arry items that are not strings', () => {
+  it('returns false for `countries` array items that are not strings', () => {
     const report = structuredClone(VALID_REPORT);
     report.userReports[0].countries = [1, 2, 3] as unknown as string[];
     expect(isValidConnectionMetricsReport(report)).toBeFalse();
   });
+  it('returns false for `asn` field type that is not a number', () => {
+    const report = structuredClone(VALID_REPORT);
+    report.userReports[0].asn = '123' as unknown as number;
+    expect(isValidConnectionMetricsReport(report)).toBeFalse();
+  });
   it('returns false for `bytesTransferred` field type that is not a number', () => {
     const report = structuredClone(VALID_REPORT);
     report.userReports[0].bytesTransferred = '1234' as unknown as number;

src/metrics_server/connection_metrics.ts

@@ -29,6 +29,7 @@ export interface ConnectionRow {
   bytesTransferred: number;
   tunnelTimeSec?: number;
   countries?: string[];
+  asn?: number;
 }
 
 export class BigQueryConnectionsTable implements InsertableTable<ConnectionRow> {
@@ -61,6 +62,7 @@ function getConnectionRowsFromReport(report: HourlyConnectionMetricsReport): Con
         bytesTransferred: userReport.bytesTransferred,
         tunnelTimeSec: userReport.tunnelTimeSec || undefined,
         countries: userReport.countries,
+        asn: userReport.asn || undefined,
       });
     }
   }
@@ -80,39 +82,40 @@ export function isValidConnectionMetricsReport(
   testObject: any
 ): testObject is HourlyConnectionMetricsReport {
   if (!testObject) {
+    console.debug('Missing test object');
     return false;
   }
 
-  // Check that all required fields are present.
   const requiredConnectionMetricsFields = ['serverId', 'startUtcMs', 'endUtcMs', 'userReports'];
   for (const fieldName of requiredConnectionMetricsFields) {
     if (!testObject[fieldName]) {
+      console.debug(`Missing required field \`${fieldName}\``);
       return false;
     }
   }
 
-  // Check that `serverId` is a string.
   if (typeof testObject.serverId !== 'string') {
+    console.debug('Invalid `serverId`');
     return false;
   }
 
-  // Check timestamp types and that startUtcMs is not after endUtcMs.
   if (
     typeof testObject.startUtcMs !== 'number' ||
     typeof testObject.endUtcMs !== 'number' ||
     testObject.startUtcMs >= testObject.endUtcMs
   ) {
+    console.debug('Invalid `startUtcMs` and/or `endUtcMs`');
     return false;
   }
 
-  // Check that at least 1 user report has been provided.
   if (testObject.userReports.length === 0) {
+    console.debug('At least 1 user report must be provided');
     return false;
   }
 
   for (const userReport of testObject.userReports) {
-    // Check that `userId` is a string.
     if (userReport.userId && typeof userReport.userId !== 'string') {
+      console.debug('Invalid `serverId`');
       return false;
     }
 
@@ -126,6 +129,7 @@ export function isValidConnectionMetricsReport(
       userReport.bytesTransferred < 0 ||
       userReport.bytesTransferred > TERABYTE
     ) {
+      console.debug('Invalid `bytesTransferred`');
       return false;
     }
 
@@ -133,20 +137,27 @@ export function isValidConnectionMetricsReport(
       userReport.tunnelTimeSec &&
       (typeof userReport.tunnelTimeSec !== 'number' || userReport.tunnelTimeSec < 0)
     ) {
+      console.debug('Invalid `tunnelTimeSec`');
       return false;
     }
 
-    // Check that `countries` is an array of strings.
     if (userReport.countries) {
       if (!Array.isArray(userReport.countries)) {
+        console.debug('Invalid `countries`');
         return false;
       }
       for (const country of userReport.countries) {
         if (typeof country !== 'string') {
+          console.debug('Invalid `countries`');
           return false;
         }
       }
     }
+
+    if (userReport.asn && typeof userReport.asn !== 'number') {
+      console.debug('Invalid `asn`');
+      return false;
+    }
   }
 
   // Request is a valid HourlyConnectionMetricsReport.

src/metrics_server/model.ts

@@ -24,6 +24,7 @@ export interface HourlyConnectionMetricsReport {
 export interface HourlyUserConnectionMetricsReport {
   userId?: string;
   countries?: string[];
+  asn?: number;
   bytesTransferred: number;
   tunnelTimeSec?: number;
 }

src/metrics_server/test_integration.sh

@@ -57,7 +57,8 @@ cat << EOF > "${CONNECTIONS_REQUEST}"
     "countries": ["US", "NL"]
   }, {
     "bytesTransferred": ${BYTES_TRANSFERRED2},
-    "countries": ["UK"]
+    "countries": ["UK"],
+    "asn": 123
   }]
 }
 EOF
@@ -78,6 +79,7 @@ EOF
 cat << EOF > "${CONNECTIONS_EXPECTED_RESPONSE}"
 [
   {
+    "asn": null,
     "bytesTransferred": "${BYTES_TRANSFERRED1}",
     "countries": [
       "US",
@@ -87,6 +89,7 @@ cat << EOF > "${CONNECTIONS_EXPECTED_RESPONSE}"
     "tunnelTimeSec": "${TUNNEL_TIME}"
   },
   {
+    "asn": "123",
     "bytesTransferred": "${BYTES_TRANSFERRED2}",
     "countries": [
       "UK"
@@ -113,7 +116,7 @@ echo "Connections request:"
 cat "${CONNECTIONS_REQUEST}"
 curl -X POST -H "Content-Type: application/json" -d "@${CONNECTIONS_REQUEST}" "${METRICS_URL}/connections" && echo
 sleep 5
-bq --project_id "${BIGQUERY_PROJECT}" --format json query --nouse_legacy_sql "SELECT serverId, bytesTransferred, tunnelTimeSec, countries FROM \`${BIGQUERY_DATASET}.${CONNECTIONS_TABLE}\` WHERE serverId = \"${SERVER_ID}\" ORDER BY bytesTransferred DESC LIMIT 2" | jq > "${CONNECTIONS_RESPONSE}"
+bq --project_id "${BIGQUERY_PROJECT}" --format json query --nouse_legacy_sql "SELECT serverId, bytesTransferred, tunnelTimeSec, countries, asn FROM \`${BIGQUERY_DATASET}.${CONNECTIONS_TABLE}\` WHERE serverId = \"${SERVER_ID}\" ORDER BY bytesTransferred DESC LIMIT 2" | jq > "${CONNECTIONS_RESPONSE}"
 diff "${CONNECTIONS_RESPONSE}" "${CONNECTIONS_EXPECTED_RESPONSE}"
 
 echo "Features request:"