CERBERUS/outline-server
1
0
Fork 0
mirror of https://github.com/OutlineFoundation/outline-server.git synced 2026-10-24 21:41:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Remove legacy metrics

Browse source
This commit is contained in:
fortuna 2019-01-04 14:37:52 -05:00
parent ce60417845
commit 1c512e2df3
2 changed files with 3 additions and 130 deletions
Download patch file Download diff file Expand all files Collapse all files

34
src/shadowbox/server/main.ts
View file

@ -27,7 +27,7 @@ import {PrometheusClient, runPrometheusScraper} from '../infrastructure/promethe
import {RolloutTracker} from '../infrastructure/rollout';
import {AccessKeyId} from '../model/access_key';
import {LegacyManagerMetrics, LegacyManagerMetricsJson, PrometheusManagerMetrics} from './manager_metrics';
import {PrometheusManagerMetrics} from './manager_metrics';
import {bindService, ShadowsocksManagerService} from './manager_service';
import {OutlineShadowsocksServer} from './outline_shadowsocks_server';
import {AccessKeyConfigJson, ServerAccessKeyRepository} from './server_access_key';
@ -35,28 +35,8 @@ import * as server_config from './server_config';
import {OutlineSharedMetricsPublisher, PrometheusUsageMetrics, RestMetricsCollectorClient, SharedMetricsPublisher} from './shared_metrics';
const DEFAULT_STATE_DIR = '/root/shadowbox/persisted-state';
const MAX_STATS_FILE_AGE_MS = 5000;
const MMDB_LOCATION = '/var/lib/libmaxminddb/GeoLite2-Country.mmdb';
// Serialized format for the metrics file.
// WARNING: Renaming fields will break backwards-compatibility.
interface MetricsConfigJson {
// Serialized ManagerStats object.
transferStats?: LegacyManagerMetricsJson;
// DEPRECATED: hourlyMetrics. Hourly stats live in memory only now.
}
function readMetricsConfig(filename: string): json_config.JsonConfig<MetricsConfigJson> {
try {
const metricsConfig = json_config.loadFileConfig<MetricsConfigJson>(filename);
// Make sure we have non-empty sub-configs.
metricsConfig.data().transferStats =
metricsConfig.data().transferStats || {} as LegacyManagerMetricsJson;
return new json_config.DelayedConfig(metricsConfig, MAX_STATS_FILE_AGE_MS);
} catch (error) {
throw new Error(`Failed to read metrics config at ${filename}: ${error}`);
}
}
async function exportPrometheusMetrics(registry: prometheus.Registry, port): Promise<http.Server> {
return new Promise<http.Server>((resolve, _) => {
@ -78,14 +58,6 @@ function reserveAccessKeyPorts(
dedupedPorts.forEach(p => portProvider.addReservedPort(p));
}
// TODO: Get rid of this after 30 days of everyone's migration to Prometheus.
function createLegacyManagerMetrics(configFilename: string): LegacyManagerMetrics {
const metricsConfig = readMetricsConfig(configFilename);
return new LegacyManagerMetrics(
new RealClock(),
new json_config.ChildConfig(metricsConfig, metricsConfig.data().transferStats));
}
function createRolloutTracker(serverConfig: json_config.JsonConfig<server_config.ServerConfigJson>):
RolloutTracker {
const rollouts = new RolloutTracker(serverConfig.data().serverId);
@ -192,9 +164,7 @@ async function main() {
const toMetricsId = (id: AccessKeyId) => {
return accessKeyRepository.getMetricsId(id);
};
const legacyManagerMetrics =
createLegacyManagerMetrics(getPersistentFilename('shadowbox_stats.json'));
const managerMetrics = new PrometheusManagerMetrics(prometheusClient, legacyManagerMetrics);
const managerMetrics = new PrometheusManagerMetrics(prometheusClient);
const metricsCollector = new RestMetricsCollectorClient(metricsCollectorUrl);
const metricsPublisher: SharedMetricsPublisher = new OutlineSharedMetricsPublisher(
new RealClock(), serverConfig, metricsReader, toMetricsId, metricsCollector);

99
src/shadowbox/server/manager_metrics.ts
View file

@ -12,19 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
import {Clock} from '../infrastructure/clock';
import {JsonConfig} from '../infrastructure/json_config';
import {PrometheusClient} from '../infrastructure/prometheus_scraper';
import {AccessKeyId} from '../model/access_key';
import {DataUsageByUser} from '../model/metrics';
export interface ManagerMetrics { get30DayByteTransfer(): Promise<DataUsageByUser>; }
// Reads manager metrics from a Prometheus instance.
export class PrometheusManagerMetrics implements ManagerMetrics {
constructor(
private prometheusClient: PrometheusClient,
private legacyManagerMetrics: LegacyManagerMetrics) {}
constructor(private prometheusClient: PrometheusClient) {}
async get30DayByteTransfer(): Promise<DataUsageByUser> {
// TODO(fortuna): Consider pre-computing this to save server's CPU.
@ -41,98 +36,6 @@ export class PrometheusManagerMetrics implements ManagerMetrics {
}
usage[entry.metric['access_key'] || ''] = bytes;
}
// TODO: Remove this after 30 days of everyone being migrated, since we won't need the config
// file anymore.
await this.addLegacyUsageData(usage);
return {bytesTransferredByUserId: usage};
}
private async addLegacyUsageData(usage: {[userId: string]: number}) {
const bytesTransferredByUserId =
(await this.legacyManagerMetrics.get30DayByteTransfer()).bytesTransferredByUserId;
for (const userId of Object.keys(bytesTransferredByUserId)) {
const oldBytes = usage[userId] || 0;
const increment = bytesTransferredByUserId[userId];
if (increment > 0) {
usage[userId] = oldBytes + increment;
}
}
}
}
// Serialized format for the manager metrics.
// WARNING: Renaming fields will break backwards-compatibility.
export interface LegacyManagerMetricsJson {
// Bytes per user per day. The key encodes the user+day in the form "userId-dateInYYYYMMDD".
dailyUserBytesTransferred?: Array<[string, number]>;
// Set of all User IDs for whom we have transfer metrics.
// TODO: Delete userIdSet. It can be derived from dailyUserBytesTransferred.
userIdSet?: string[];
}
// ManagerMetrics keeps track of the number of bytes transferred per user, per day.
// Surfaced by the manager service to display on the Manager UI.
// TODO: Remove entries older than 30d.
export class LegacyManagerMetrics implements ManagerMetrics {
private dailyUserBytesTransferred: Map<string, number>;
private userIdSet: Set<AccessKeyId>;
constructor(private clock: Clock, private config: JsonConfig<LegacyManagerMetricsJson>) {
const serializedObject = config.data();
if (serializedObject) {
this.dailyUserBytesTransferred = new Map(serializedObject.dailyUserBytesTransferred);
this.userIdSet = new Set(serializedObject.userIdSet);
} else {
this.dailyUserBytesTransferred = new Map();
this.userIdSet = new Set();
}
}
writeBytesTransferred(userId: AccessKeyId, numBytes: number) {
this.userIdSet.add(userId);
const date = new Date(this.clock.now());
const oldTotal = this.getBytes(userId, date);
const newTotal = oldTotal + numBytes;
this.dailyUserBytesTransferred.set(this.getKey(userId, date), newTotal);
this.toJson(this.config.data());
this.config.write();
}
get30DayByteTransfer(): Promise<DataUsageByUser> {
const bytesTransferredByUserId = {};
for (let i = 0; i < 30; ++i) {
// Get Date from i days ago.
const d = new Date(this.clock.now());
d.setDate(d.getDate() - i);
// Get transfer per userId and total
for (const userId of this.userIdSet) {
if (!bytesTransferredByUserId[userId]) {
bytesTransferredByUserId[userId] = 0;
}
const numBytes = this.getBytes(userId, d);
bytesTransferredByUserId[userId] += numBytes;
}
}
return Promise.resolve({bytesTransferredByUserId});
}
// Returns the state of this object, e.g.
// {"dailyUserBytesTransferred":[["0-20170816",100],["1-20170816",100]],"userIdSet":["0","1"]}
private toJson(target: LegacyManagerMetricsJson) {
// Use [...] operator to serialize Map and Set objects to JSON.
target.dailyUserBytesTransferred = [...this.dailyUserBytesTransferred];
target.userIdSet = [...this.userIdSet];
}
private getBytes(userId: AccessKeyId, d: Date) {
const key = this.getKey(userId, d);
return this.dailyUserBytesTransferred.get(key) || 0;
}
private getKey(userId: AccessKeyId, d: Date) {
const yyyymmdd = d.toISOString().substr(0, 'YYYY-MM-DD'.length).replace(/-/g, '');
return `${userId}-${yyyymmdd}`;
}
}