From 97936beccfb37b86f8baec068de1d7805285f010 Mon Sep 17 00:00:00 2001 From: sbruens Date: Thu, 16 May 2024 10:18:43 -0400 Subject: [PATCH 1/3] Rename `CountryUsage` to `LocationUsage` so it can encompass ASN. --- src/shadowbox/server/shared_metrics.spec.ts | 8 ++++---- src/shadowbox/server/shared_metrics.ts | 22 ++++++++++----------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/shadowbox/server/shared_metrics.spec.ts b/src/shadowbox/server/shared_metrics.spec.ts index 3798df9e1..b70ce102e 100644 --- a/src/shadowbox/server/shared_metrics.spec.ts +++ b/src/shadowbox/server/shared_metrics.spec.ts @@ -22,7 +22,7 @@ import {AccessKeyConfigJson} from './server_access_key'; import {ServerConfigJson} from './server_config'; import { - CountryUsage, + LocationUsage, DailyFeatureMetricsReportJson, HourlyServerMetricsReportJson, MetricsCollectorClient, @@ -222,14 +222,14 @@ class FakeMetricsCollector implements MetricsCollectorClient { } class ManualUsageMetrics implements UsageMetrics { - public countryUsage = [] as CountryUsage[]; + public countryUsage = [] as LocationUsage[]; - getCountryUsage(): Promise { + getLocationUsage(): Promise { return Promise.resolve(this.countryUsage); } reset() { - this.countryUsage = [] as CountryUsage[]; + this.countryUsage = [] as LocationUsage[]; } } diff --git a/src/shadowbox/server/shared_metrics.ts b/src/shadowbox/server/shared_metrics.ts index a19098ed5..9f6a816a0 100644 --- a/src/shadowbox/server/shared_metrics.ts +++ b/src/shadowbox/server/shared_metrics.ts @@ -26,7 +26,7 @@ const MS_PER_HOUR = 60 * 60 * 1000; const MS_PER_DAY = 24 * MS_PER_HOUR; const SANCTIONED_COUNTRIES = new Set(['CU', 'KP', 'SY']); -export interface CountryUsage { +export interface LocationUsage { country: string; inboundBytes: number; } @@ -70,7 +70,7 @@ export interface SharedMetricsPublisher { } export interface UsageMetrics { - getCountryUsage(): Promise; + getLocationUsage(): Promise; reset(); } @@ -80,13 +80,13 @@ export class PrometheusUsageMetrics implements UsageMetrics { constructor(private prometheusClient: PrometheusClient) {} - async getCountryUsage(): Promise { + async getLocationUsage(): Promise { const timeDeltaSecs = Math.round((Date.now() - this.resetTimeMs) / 1000); // We measure the traffic to and from the target, since that's what we are protecting. const result = await this.prometheusClient.query( `sum(increase(shadowsocks_data_bytes_per_location{dir=~"p>t|p { + private async reportServerUsageMetrics(locationUsageMetrics: LocationUsage[]): Promise { const reportEndTimestampMs = this.clock.now(); const userReports = [] as HourlyUserMetricsReportJson[]; - for (const countryUsage of countryUsageMetrics) { - if (countryUsage.inboundBytes === 0) { + for (const locationUsage of locationUsageMetrics) { + if (locationUsage.inboundBytes === 0) { continue; } - if (isSanctionedCountry(countryUsage.country)) { + if (isSanctionedCountry(locationUsage.country)) { continue; } // Make sure to always set a country, which is required by the metrics server validation. // It's used to differentiate the row from the legacy key usage rows. - const country = countryUsage.country || 'ZZ'; + const country = locationUsage.country || 'ZZ'; userReports.push({ - bytesTransferred: countryUsage.inboundBytes, + bytesTransferred: locationUsage.inboundBytes, countries: [country], }); } From a3ae7910acadcbcac21c8212621aa19c4c727e6e Mon Sep 17 00:00:00 2001 From: sbruens Date: Mon, 20 May 2024 16:28:06 -0400 Subject: [PATCH 2/3] Add ASN metric to opt-in server usage report. --- src/shadowbox/server/shared_metrics.spec.ts | 15 +++++++++++++++ src/shadowbox/server/shared_metrics.ts | 17 ++++++++++++----- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/src/shadowbox/server/shared_metrics.spec.ts b/src/shadowbox/server/shared_metrics.spec.ts index b70ce102e..30047ce59 100644 --- a/src/shadowbox/server/shared_metrics.spec.ts +++ b/src/shadowbox/server/shared_metrics.spec.ts @@ -114,6 +114,21 @@ describe('OutlineSharedMetricsPublisher', () => { }); }); + it('sends ASN data if present', async () => { + usageMetrics.countryUsage = [ + {country: 'DD', asn: 999, inboundBytes: 44}, + {country: 'EE', inboundBytes: 55}, + ]; + clock.nowMs += 60 * 60 * 1000; + + await clock.runCallbacks(); + + expect(metricsCollector.collectedServerUsageReport.userReports).toEqual([ + {bytesTransferred: 44, countries: ['DD'], asn: 999}, + {bytesTransferred: 55, countries: ['EE']}, + ]); + }); + it('resets metrics to avoid double reporting', async () => { usageMetrics.countryUsage = [ {country: 'AA', inboundBytes: 11}, diff --git a/src/shadowbox/server/shared_metrics.ts b/src/shadowbox/server/shared_metrics.ts index 9f6a816a0..5e7f8c4fd 100644 --- a/src/shadowbox/server/shared_metrics.ts +++ b/src/shadowbox/server/shared_metrics.ts @@ -28,6 +28,7 @@ const SANCTIONED_COUNTRIES = new Set(['CU', 'KP', 'SY']); export interface LocationUsage { country: string; + asn?: number; inboundBytes: number; } @@ -44,6 +45,7 @@ export interface HourlyServerMetricsReportJson { // Field renames will break backwards-compatibility. export interface HourlyUserMetricsReportJson { countries: string[]; + asn?: number; bytesTransferred: number; } @@ -84,13 +86,14 @@ export class PrometheusUsageMetrics implements UsageMetrics { const timeDeltaSecs = Math.round((Date.now() - this.resetTimeMs) / 1000); // We measure the traffic to and from the target, since that's what we are protecting. const result = await this.prometheusClient.query( - `sum(increase(shadowsocks_data_bytes_per_location{dir=~"p>t|pt|p { const reportEndTimestampMs = this.clock.now(); - const userReports = [] as HourlyUserMetricsReportJson[]; + const userReports: HourlyUserMetricsReportJson[] = []; for (const locationUsage of locationUsageMetrics) { if (locationUsage.inboundBytes === 0) { continue; @@ -211,10 +214,14 @@ export class OutlineSharedMetricsPublisher implements SharedMetricsPublisher { // Make sure to always set a country, which is required by the metrics server validation. // It's used to differentiate the row from the legacy key usage rows. const country = locationUsage.country || 'ZZ'; - userReports.push({ + const report: HourlyUserMetricsReportJson = { bytesTransferred: locationUsage.inboundBytes, countries: [country], - }); + }; + if (locationUsage.asn) { + report.asn = locationUsage.asn; + } + userReports.push(report); } const report = { serverId: this.serverConfig.data().serverId, From 59815f8a4595db9563de5c11ac4a9b68784a4d39 Mon Sep 17 00:00:00 2001 From: sbruens Date: Tue, 21 May 2024 14:46:24 -0400 Subject: [PATCH 3/3] Rename `countryUsage` in tests as well, for consistency. --- src/shadowbox/server/shared_metrics.spec.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/shadowbox/server/shared_metrics.spec.ts b/src/shadowbox/server/shared_metrics.spec.ts index 30047ce59..ba5622433 100644 --- a/src/shadowbox/server/shared_metrics.spec.ts +++ b/src/shadowbox/server/shared_metrics.spec.ts @@ -89,7 +89,7 @@ describe('OutlineSharedMetricsPublisher', () => { describe('for server usage', () => { it('is sending correct reports', async () => { - usageMetrics.countryUsage = [ + usageMetrics.locationUsage = [ {country: 'AA', inboundBytes: 11}, {country: 'BB', inboundBytes: 11}, {country: 'CC', inboundBytes: 22}, @@ -115,7 +115,7 @@ describe('OutlineSharedMetricsPublisher', () => { }); it('sends ASN data if present', async () => { - usageMetrics.countryUsage = [ + usageMetrics.locationUsage = [ {country: 'DD', asn: 999, inboundBytes: 44}, {country: 'EE', inboundBytes: 55}, ]; @@ -130,15 +130,15 @@ describe('OutlineSharedMetricsPublisher', () => { }); it('resets metrics to avoid double reporting', async () => { - usageMetrics.countryUsage = [ + usageMetrics.locationUsage = [ {country: 'AA', inboundBytes: 11}, {country: 'BB', inboundBytes: 11}, ]; clock.nowMs += 60 * 60 * 1000; startTime = clock.nowMs; await clock.runCallbacks(); - usageMetrics.countryUsage = [ - ...usageMetrics.countryUsage, + usageMetrics.locationUsage = [ + ...usageMetrics.locationUsage, {country: 'CC', inboundBytes: 22}, {country: 'DD', inboundBytes: 22}, ]; @@ -153,7 +153,7 @@ describe('OutlineSharedMetricsPublisher', () => { }); it('ignores sanctioned countries', async () => { - usageMetrics.countryUsage = [ + usageMetrics.locationUsage = [ {country: 'AA', inboundBytes: 11}, {country: 'SY', inboundBytes: 11}, {country: 'CC', inboundBytes: 22}, @@ -237,14 +237,14 @@ class FakeMetricsCollector implements MetricsCollectorClient { } class ManualUsageMetrics implements UsageMetrics { - public countryUsage = [] as LocationUsage[]; + public locationUsage = [] as LocationUsage[]; getLocationUsage(): Promise { - return Promise.resolve(this.countryUsage); + return Promise.resolve(this.locationUsage); } reset() { - this.countryUsage = [] as LocationUsage[]; + this.locationUsage = [] as LocationUsage[]; } }