diff --git a/src/shadowbox/server/main.ts b/src/shadowbox/server/main.ts index 010b4f52..8a903dc3 100644 --- a/src/shadowbox/server/main.ts +++ b/src/shadowbox/server/main.ts @@ -27,7 +27,7 @@ import {PrometheusClient, runPrometheusScraper} from '../infrastructure/promethe import {RolloutTracker} from '../infrastructure/rollout'; import {AccessKeyId} from '../model/access_key'; -import {LegacyManagerMetrics, LegacyManagerMetricsJson, PrometheusManagerMetrics} from './manager_metrics'; +import {PrometheusManagerMetrics} from './manager_metrics'; import {bindService, ShadowsocksManagerService} from './manager_service'; import {OutlineShadowsocksServer} from './outline_shadowsocks_server'; import {AccessKeyConfigJson, ServerAccessKeyRepository} from './server_access_key'; @@ -35,28 +35,8 @@ import * as server_config from './server_config'; import {OutlineSharedMetricsPublisher, PrometheusUsageMetrics, RestMetricsCollectorClient, SharedMetricsPublisher} from './shared_metrics'; const DEFAULT_STATE_DIR = '/root/shadowbox/persisted-state'; -const MAX_STATS_FILE_AGE_MS = 5000; const MMDB_LOCATION = '/var/lib/libmaxminddb/GeoLite2-Country.mmdb'; -// Serialized format for the metrics file. -// WARNING: Renaming fields will break backwards-compatibility. -interface MetricsConfigJson { - // Serialized ManagerStats object. - transferStats?: LegacyManagerMetricsJson; - // DEPRECATED: hourlyMetrics. Hourly stats live in memory only now. -} - -function readMetricsConfig(filename: string): json_config.JsonConfig { - try { - const metricsConfig = json_config.loadFileConfig(filename); - // Make sure we have non-empty sub-configs. - metricsConfig.data().transferStats = - metricsConfig.data().transferStats || {} as LegacyManagerMetricsJson; - return new json_config.DelayedConfig(metricsConfig, MAX_STATS_FILE_AGE_MS); - } catch (error) { - throw new Error(`Failed to read metrics config at ${filename}: ${error}`); - } -} async function exportPrometheusMetrics(registry: prometheus.Registry, port): Promise { return new Promise((resolve, _) => { @@ -78,14 +58,6 @@ function reserveAccessKeyPorts( dedupedPorts.forEach(p => portProvider.addReservedPort(p)); } -// TODO: Get rid of this after 30 days of everyone's migration to Prometheus. -function createLegacyManagerMetrics(configFilename: string): LegacyManagerMetrics { - const metricsConfig = readMetricsConfig(configFilename); - return new LegacyManagerMetrics( - new RealClock(), - new json_config.ChildConfig(metricsConfig, metricsConfig.data().transferStats)); -} - function createRolloutTracker(serverConfig: json_config.JsonConfig): RolloutTracker { const rollouts = new RolloutTracker(serverConfig.data().serverId); @@ -192,9 +164,7 @@ async function main() { const toMetricsId = (id: AccessKeyId) => { return accessKeyRepository.getMetricsId(id); }; - const legacyManagerMetrics = - createLegacyManagerMetrics(getPersistentFilename('shadowbox_stats.json')); - const managerMetrics = new PrometheusManagerMetrics(prometheusClient, legacyManagerMetrics); + const managerMetrics = new PrometheusManagerMetrics(prometheusClient); const metricsCollector = new RestMetricsCollectorClient(metricsCollectorUrl); const metricsPublisher: SharedMetricsPublisher = new OutlineSharedMetricsPublisher( new RealClock(), serverConfig, metricsReader, toMetricsId, metricsCollector); diff --git a/src/shadowbox/server/manager_metrics.ts b/src/shadowbox/server/manager_metrics.ts index 8995b68d..ca2985b3 100644 --- a/src/shadowbox/server/manager_metrics.ts +++ b/src/shadowbox/server/manager_metrics.ts @@ -12,19 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -import {Clock} from '../infrastructure/clock'; -import {JsonConfig} from '../infrastructure/json_config'; import {PrometheusClient} from '../infrastructure/prometheus_scraper'; -import {AccessKeyId} from '../model/access_key'; import {DataUsageByUser} from '../model/metrics'; export interface ManagerMetrics { get30DayByteTransfer(): Promise; } // Reads manager metrics from a Prometheus instance. export class PrometheusManagerMetrics implements ManagerMetrics { - constructor( - private prometheusClient: PrometheusClient, - private legacyManagerMetrics: LegacyManagerMetrics) {} + constructor(private prometheusClient: PrometheusClient) {} async get30DayByteTransfer(): Promise { // TODO(fortuna): Consider pre-computing this to save server's CPU. @@ -41,98 +36,6 @@ export class PrometheusManagerMetrics implements ManagerMetrics { } usage[entry.metric['access_key'] || ''] = bytes; } - // TODO: Remove this after 30 days of everyone being migrated, since we won't need the config - // file anymore. - await this.addLegacyUsageData(usage); return {bytesTransferredByUserId: usage}; } - - private async addLegacyUsageData(usage: {[userId: string]: number}) { - const bytesTransferredByUserId = - (await this.legacyManagerMetrics.get30DayByteTransfer()).bytesTransferredByUserId; - for (const userId of Object.keys(bytesTransferredByUserId)) { - const oldBytes = usage[userId] || 0; - const increment = bytesTransferredByUserId[userId]; - if (increment > 0) { - usage[userId] = oldBytes + increment; - } - } - } -} - -// Serialized format for the manager metrics. -// WARNING: Renaming fields will break backwards-compatibility. -export interface LegacyManagerMetricsJson { - // Bytes per user per day. The key encodes the user+day in the form "userId-dateInYYYYMMDD". - dailyUserBytesTransferred?: Array<[string, number]>; - // Set of all User IDs for whom we have transfer metrics. - // TODO: Delete userIdSet. It can be derived from dailyUserBytesTransferred. - userIdSet?: string[]; -} - -// ManagerMetrics keeps track of the number of bytes transferred per user, per day. -// Surfaced by the manager service to display on the Manager UI. -// TODO: Remove entries older than 30d. -export class LegacyManagerMetrics implements ManagerMetrics { - private dailyUserBytesTransferred: Map; - private userIdSet: Set; - - constructor(private clock: Clock, private config: JsonConfig) { - const serializedObject = config.data(); - if (serializedObject) { - this.dailyUserBytesTransferred = new Map(serializedObject.dailyUserBytesTransferred); - this.userIdSet = new Set(serializedObject.userIdSet); - } else { - this.dailyUserBytesTransferred = new Map(); - this.userIdSet = new Set(); - } - } - - writeBytesTransferred(userId: AccessKeyId, numBytes: number) { - this.userIdSet.add(userId); - - const date = new Date(this.clock.now()); - const oldTotal = this.getBytes(userId, date); - const newTotal = oldTotal + numBytes; - this.dailyUserBytesTransferred.set(this.getKey(userId, date), newTotal); - this.toJson(this.config.data()); - this.config.write(); - } - - get30DayByteTransfer(): Promise { - const bytesTransferredByUserId = {}; - for (let i = 0; i < 30; ++i) { - // Get Date from i days ago. - const d = new Date(this.clock.now()); - d.setDate(d.getDate() - i); - - // Get transfer per userId and total - for (const userId of this.userIdSet) { - if (!bytesTransferredByUserId[userId]) { - bytesTransferredByUserId[userId] = 0; - } - const numBytes = this.getBytes(userId, d); - bytesTransferredByUserId[userId] += numBytes; - } - } - return Promise.resolve({bytesTransferredByUserId}); - } - - // Returns the state of this object, e.g. - // {"dailyUserBytesTransferred":[["0-20170816",100],["1-20170816",100]],"userIdSet":["0","1"]} - private toJson(target: LegacyManagerMetricsJson) { - // Use [...] operator to serialize Map and Set objects to JSON. - target.dailyUserBytesTransferred = [...this.dailyUserBytesTransferred]; - target.userIdSet = [...this.userIdSet]; - } - - private getBytes(userId: AccessKeyId, d: Date) { - const key = this.getKey(userId, d); - return this.dailyUserBytesTransferred.get(key) || 0; - } - - private getKey(userId: AccessKeyId, d: Date) { - const yyyymmdd = d.toISOString().substr(0, 'YYYY-MM-DD'.length).replace(/-/g, ''); - return `${userId}-${yyyymmdd}`; - } }