diff --git a/.docker/development/docker-compose.yml b/.docker/development/docker-compose.yml index c458aa9..e094d89 100644 --- a/.docker/development/docker-compose.yml +++ b/.docker/development/docker-compose.yml @@ -12,6 +12,8 @@ services: POSTGRES_PASSWORD: postgres volumes: - 'postgres_data:/var/lib/postgresql/data' + redis: + image: redis # custom services backend: diff --git a/src/config/fragments/docker.ts b/src/config/fragments/docker.ts index 3c57696..43ee4ae 100644 --- a/src/config/fragments/docker.ts +++ b/src/config/fragments/docker.ts @@ -4,4 +4,8 @@ export const dockerFragment: FragmentSchema = { postgres: { connection: 'postgres://postgres:postgres@postgres:5432/postgres', }, + ratelimits: { + enabled: true, + redisUrl: 'redis://redis:6379', + }, }; diff --git a/src/config/orm.ts b/src/config/orm.ts index 5116bad..7412ee7 100644 --- a/src/config/orm.ts +++ b/src/config/orm.ts @@ -1,6 +1,13 @@ +import { devFragment } from '@/config/fragments/dev'; +import { dockerFragment } from '@/config/fragments/docker'; import { createConfigLoader } from 'neat-config'; import { z } from 'zod'; +const fragments = { + dev: devFragment, + dockerdev: dockerFragment, +}; + export const ormConfigSchema = z.object({ postgres: z.object({ // connection URL for postgres database @@ -15,6 +22,8 @@ export const ormConf = createConfigLoader() prefix: 'MWB_', }) .addFromFile('config.json') + .setFragmentKey('usePresets') + .addConfigFragments(fragments) .addZodSchema(ormConfigSchema) .freeze() .load(); diff --git a/src/config/schema.ts b/src/config/schema.ts index e6ec194..b4ad626 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -16,6 +16,9 @@ export const configSchema = z.object({ // should it trust reverse proxy headers? (for ip gathering) trustProxy: z.coerce.boolean().default(false), + // should it trust cloudflare headers? (for ip gathering, cloudflare has priority) + trustCloudflare: z.coerce.boolean().default(false), + // prefix for where the instance is run on. for example set it to /backend if you're hosting it on example.com/backend // if this is set, do not apply url rewriting before proxing basePath: z.string().default('/'), diff --git a/src/db/migrations/.snapshot-movie_web.json b/src/db/migrations/.snapshot-postgres.json similarity index 77% rename from src/db/migrations/.snapshot-movie_web.json rename to src/db/migrations/.snapshot-postgres.json index 4a6d346..b4204ac 100644 --- a/src/db/migrations/.snapshot-movie_web.json +++ b/src/db/migrations/.snapshot-postgres.json @@ -268,143 +268,6 @@ "checks": [], "foreignKeys": {} }, - { - "columns": { - "id": { - "name": "id", - "type": "uuid", - "unsigned": false, - "autoincrement": false, - "primary": false, - "nullable": false, - "mappedType": "uuid" - }, - "tmdb_id": { - "name": "tmdb_id", - "type": "varchar(255)", - "unsigned": false, - "autoincrement": false, - "primary": false, - "nullable": false, - "mappedType": "string" - }, - "type": { - "name": "type", - "type": "varchar(255)", - "unsigned": false, - "autoincrement": false, - "primary": false, - "nullable": false, - "mappedType": "string" - }, - "title": { - "name": "title", - "type": "varchar(255)", - "unsigned": false, - "autoincrement": false, - "primary": false, - "nullable": false, - "mappedType": "string" - }, - "season_id": { - "name": "season_id", - "type": "varchar(255)", - "unsigned": false, - "autoincrement": false, - "primary": false, - "nullable": true, - "mappedType": "string" - }, - "episode_id": { - "name": "episode_id", - "type": "varchar(255)", - "unsigned": false, - "autoincrement": false, - "primary": false, - "nullable": true, - "mappedType": "string" - }, - "created_at": { - "name": "created_at", - "type": "timestamptz(0)", - "unsigned": false, - "autoincrement": false, - "primary": false, - "nullable": false, - "length": 0, - "mappedType": "datetime" - }, - "status": { - "name": "status", - "type": "varchar(255)", - "unsigned": false, - "autoincrement": false, - "primary": false, - "nullable": false, - "mappedType": "string" - }, - "provider_id": { - "name": "provider_id", - "type": "varchar(255)", - "unsigned": false, - "autoincrement": false, - "primary": false, - "nullable": false, - "mappedType": "string" - }, - "embed_id": { - "name": "embed_id", - "type": "varchar(255)", - "unsigned": false, - "autoincrement": false, - "primary": false, - "nullable": true, - "mappedType": "string" - }, - "error_message": { - "name": "error_message", - "type": "text", - "unsigned": false, - "autoincrement": false, - "primary": false, - "nullable": true, - "mappedType": "text" - }, - "full_error": { - "name": "full_error", - "type": "text", - "unsigned": false, - "autoincrement": false, - "primary": false, - "nullable": true, - "mappedType": "text" - }, - "hostname": { - "name": "hostname", - "type": "varchar(255)", - "unsigned": false, - "autoincrement": false, - "primary": false, - "nullable": false, - "mappedType": "string" - } - }, - "name": "provider_metrics", - "schema": "public", - "indexes": [ - { - "keyName": "provider_metrics_pkey", - "columnNames": [ - "id" - ], - "composite": false, - "primary": true, - "unique": true - } - ], - "checks": [], - "foreignKeys": {} - }, { "columns": { "id": { diff --git a/src/db/migrations/Migration20231221185725.ts b/src/db/migrations/Migration20231221185725.ts new file mode 100644 index 0000000..0993254 --- /dev/null +++ b/src/db/migrations/Migration20231221185725.ts @@ -0,0 +1,13 @@ +import { Migration } from '@mikro-orm/migrations'; + +export class Migration20231221185725 extends Migration { + + async up(): Promise { + this.addSql('drop table if exists "provider_metrics" cascade;'); + } + + async down(): Promise { + this.addSql('create table "provider_metrics" ("id" uuid not null default null, "tmdb_id" varchar not null default null, "type" varchar not null default null, "title" varchar not null default null, "season_id" varchar null default null, "episode_id" varchar null default null, "created_at" timestamptz not null default null, "status" varchar not null default null, "provider_id" varchar not null default null, "embed_id" varchar null default null, "error_message" text null default null, "full_error" text null default null, "hostname" varchar not null default null, constraint "provider_metrics_pkey" primary key ("id"));'); + } + +} diff --git a/src/db/models/ProviderMetrics.ts b/src/db/models/ProviderMetrics.ts deleted file mode 100644 index 4a84ce0..0000000 --- a/src/db/models/ProviderMetrics.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { Entity, PrimaryKey, Property } from '@mikro-orm/core'; -import { randomUUID } from 'crypto'; - -export const status = { - failed: 'failed', - notfound: 'notfound', - success: 'success', -} as const; -type Status = keyof typeof status; - -@Entity({ tableName: 'provider_metrics' }) -export class ProviderMetric { - @PrimaryKey({ name: 'id', type: 'uuid' }) - id: string = randomUUID(); - - @Property({ name: 'tmdb_id' }) - tmdbId!: string; - - @Property({ name: 'type' }) - type!: string; - - @Property({ name: 'title' }) - title!: string; - - @Property({ name: 'season_id', nullable: true }) - seasonId?: string; - - @Property({ name: 'episode_id', nullable: true }) - episodeId?: string; - - @Property({ name: 'created_at', type: 'date' }) - createdAt = new Date(); - - @Property({ name: 'status' }) - status!: Status; - - @Property({ name: 'provider_id' }) - providerId!: string; - - @Property({ name: 'embed_id', nullable: true }) - embedId?: string; - - @Property({ name: 'error_message', nullable: true, type: 'text' }) - errorMessage?: string; - - @Property({ name: 'full_error', nullable: true, type: 'text' }) - fullError?: string; - - @Property({ name: 'hostname' }) - hostname!: string; -} diff --git a/src/modules/jobs/index.ts b/src/modules/jobs/index.ts index 6258549..e6e955d 100644 --- a/src/modules/jobs/index.ts +++ b/src/modules/jobs/index.ts @@ -1,11 +1,9 @@ import { challengeCodeJob } from '@/modules/jobs/list/challengeCode'; import { sessionExpiryJob } from '@/modules/jobs/list/sessionExpiry'; import { userDeletionJob } from '@/modules/jobs/list/userDeletion'; -import { providerMetricCleanupJob } from '@/modules/jobs/list/providerMetricCleanup'; export async function setupJobs() { challengeCodeJob.start(); sessionExpiryJob.start(); userDeletionJob.start(); - providerMetricCleanupJob.start(); } diff --git a/src/modules/jobs/list/providerMetricCleanup.ts b/src/modules/jobs/list/providerMetricCleanup.ts deleted file mode 100644 index c2234b8..0000000 --- a/src/modules/jobs/list/providerMetricCleanup.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { ProviderMetric } from '@/db/models/ProviderMetrics'; -import { job } from '@/modules/jobs/job'; -import ms from 'ms'; - -// every day at 12:00:00 -export const providerMetricCleanupJob = job( - 'provider-metric-cleanup', - '0 12 * * *', - async ({ em, log }) => { - const now = new Date(); - const thirtyDaysAgo = new Date(now.getTime() - ms('30d')); - - const deletedMetrics = await em - .createQueryBuilder(ProviderMetric) - .delete() - .where({ - createdAt: { - $lt: thirtyDaysAgo, - }, - }) - .execute<{ affectedRows: number }>('run'); - - log.info( - `Removed ${deletedMetrics.affectedRows} metrics that were older than 30 days`, - ); - }, -); diff --git a/src/modules/metrics/index.ts b/src/modules/metrics/index.ts index 9d23d4b..e6f798f 100644 --- a/src/modules/metrics/index.ts +++ b/src/modules/metrics/index.ts @@ -9,17 +9,10 @@ const log = scopedLogger('metrics'); export type Metrics = { user: Counter<'namespace'>; - providerMetrics: Counter< - | 'title' - | 'tmdb_id' - | 'season_id' - | 'episode_id' - | 'status' - | 'type' - | 'provider_id' - | 'embed_id' - | 'hostname' - >; + captchaSolves: Counter<'success'>; + providerHostnames: Counter<'hostname'>; + providerStatuses: Counter<'provider_id' | 'status'>; + watchMetrics: Counter<'title' | 'tmdb_full_id' | 'provider_id' | 'success'>; }; let metrics: null | Metrics = null; @@ -42,31 +35,38 @@ export async function setupMetrics(app: FastifyInstance) { metrics = { user: new Counter({ - name: 'user_count', - help: 'user_help', + name: 'mw_user_count', + help: 'mw_user_help', labelNames: ['namespace'], }), - providerMetrics: new Counter({ - name: 'provider_metrics', - help: 'provider_metrics', - labelNames: [ - 'episode_id', - 'provider_id', - 'season_id', - 'status', - 'title', - 'tmdb_id', - 'type', - 'embed_id', - 'hostname', - ], + captchaSolves: new Counter({ + name: 'mw_captcha_solves', + help: 'mw_captcha_solves', + labelNames: ['success'], + }), + providerHostnames: new Counter({ + name: 'mw_provider_hostname_count', + help: 'mw_provider_hostname_count', + labelNames: ['hostname'], + }), + providerStatuses: new Counter({ + name: 'mw_provider_status_count', + help: 'mw_provider_status_count', + labelNames: ['provider_id', 'status'], + }), + watchMetrics: new Counter({ + name: 'mw_media_watch_count', + help: 'mw_media_watch_count', + labelNames: ['title', 'tmdb_full_id', 'provider_id', 'success'], }), }; const promClient = app.metrics.client; promClient.register.registerMetric(metrics.user); - promClient.register.registerMetric(metrics.providerMetrics); + promClient.register.registerMetric(metrics.providerHostnames); + promClient.register.registerMetric(metrics.providerStatuses); + promClient.register.registerMetric(metrics.watchMetrics); const orm = getORM(); const em = orm.em.fork(); diff --git a/src/modules/ratelimits/limiter.ts b/src/modules/ratelimits/limiter.ts index 628ce59..45ca519 100644 --- a/src/modules/ratelimits/limiter.ts +++ b/src/modules/ratelimits/limiter.ts @@ -2,6 +2,7 @@ import Redis from 'ioredis'; import RateLimiter from 'async-ratelimiter'; import ms from 'ms'; import { StatusError } from '@/services/error'; +import { IpReq, getIp } from '@/services/ip'; export interface LimiterOptions { redis: Redis; @@ -26,8 +27,8 @@ export class Limiter { this.redis = ops.redis; } - async bump(req: { ip: string }, ops: BucketOptions) { - const ip = req.ip; + async bump(req: IpReq, ops: BucketOptions) { + const ip = getIp(req); if (!this.buckets[ops.id]) { this.buckets[ops.id] = { limiter: new RateLimiter({ @@ -54,7 +55,7 @@ export class Limiter { }; } - async assertAndBump(req: { ip: string }, ops: BucketOptions) { + async assertAndBump(req: IpReq, ops: BucketOptions) { const { hasBeenLimited } = await this.bump(req, ops); if (hasBeenLimited) { throw new StatusError('Ratelimited', 429); diff --git a/src/routes/metrics.ts b/src/routes/metrics.ts index 899ce5e..e560505 100644 --- a/src/routes/metrics.ts +++ b/src/routes/metrics.ts @@ -1,8 +1,8 @@ import { handle } from '@/services/handler'; import { makeRouter } from '@/services/router'; import { z } from 'zod'; -import { ProviderMetric, status } from '@/db/models/ProviderMetrics'; import { getMetrics } from '@/modules/metrics'; +import { status } from '@/routes/statuses'; const metricsProviderSchema = z.object({ tmdbId: z.string(), @@ -29,7 +29,7 @@ export const metricsRouter = makeRouter((app) => { body: metricsProviderInputSchema, }, }, - handle(async ({ em, body, req, limiter }) => { + handle(async ({ body, req, limiter }) => { await limiter?.assertAndBump(req, { id: 'provider_metrics', max: 300, @@ -37,43 +37,59 @@ export const metricsRouter = makeRouter((app) => { window: '30m', }); - const hostname = req.headers.origin?.slice(0, 255) ?? 'unknown origin'; - - const entities = body.items.map((v) => { - const errorMessage = v.errorMessage?.slice(0, 200); - const truncatedFullError = v.fullError?.slice(0, 2000); - - const metric = new ProviderMetric(); - em.assign(metric, { - providerId: v.providerId, - embedId: v.embedId, - fullError: truncatedFullError, - errorMessage: errorMessage, - episodeId: v.episodeId, - seasonId: v.seasonId, - status: v.status, - title: v.title, - tmdbId: v.tmdbId, - type: v.type, - hostname, - }); - return metric; + const hostname = req.headers.origin?.slice(0, 255) ?? ''; + getMetrics().providerHostnames.inc({ + hostname, }); - entities.forEach((entity) => { - getMetrics().providerMetrics.inc({ - episode_id: entity.episodeId, - provider_id: entity.providerId, - season_id: entity.seasonId, - status: entity.status, - title: entity.title, - tmdb_id: entity.tmdbId, - type: entity.type, - hostname, + body.items.forEach((item) => { + getMetrics().providerStatuses.inc({ + provider_id: item.embedId ?? item.providerId, + status: item.status, }); }); - await em.persistAndFlush(entities); + const itemList = [...body.items]; + itemList.reverse(); + const lastSuccessfulItem = body.items.find( + (v) => v.status === status.success, + ); + const lastItem = itemList[0]; + + if (lastItem) { + getMetrics().watchMetrics.inc({ + tmdb_full_id: lastItem.type + '-' + lastItem.tmdbId, + provider_id: lastSuccessfulItem?.providerId ?? lastItem.providerId, + title: lastItem.title, + success: (!!lastSuccessfulItem).toString(), + }); + } + + return true; + }), + ); + + app.post( + '/metrics/captcha', + { + schema: { + body: z.object({ + success: z.boolean(), + }), + }, + }, + handle(async ({ body, req, limiter }) => { + await limiter?.assertAndBump(req, { + id: 'captcha_solves', + max: 300, + inc: 1, + window: '30m', + }); + + getMetrics().captchaSolves.inc({ + success: body.success.toString(), + }); + return true; }), ); diff --git a/src/routes/statuses.ts b/src/routes/statuses.ts new file mode 100644 index 0000000..3b353f8 --- /dev/null +++ b/src/routes/statuses.ts @@ -0,0 +1,6 @@ +export const status = { + failed: 'failed', + notfound: 'notfound', + success: 'success', +} as const; +export type Status = keyof typeof status; diff --git a/src/services/ip.ts b/src/services/ip.ts new file mode 100644 index 0000000..adf32c6 --- /dev/null +++ b/src/services/ip.ts @@ -0,0 +1,27 @@ +import { conf } from '@/config'; +import { IncomingHttpHeaders } from 'http'; + +export type IpReq = { + ip: string; + headers: IncomingHttpHeaders; +}; + +const trustCloudflare = conf.server.trustCloudflare; + +function getSingleHeader( + headers: IncomingHttpHeaders, + key: string, +): string | undefined { + const header = headers[key]; + if (Array.isArray(header)) return header[0]; + return header; +} + +export function getIp(req: IpReq) { + const cfIp = getSingleHeader(req.headers, 'cf-connecting-ip'); + if (trustCloudflare && cfIp) { + return cfIp; + } + + return req.ip; +}