dev
/
backend
mirror of https://github.com/sussy-code/backend.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #26 from movie-web/dev 

Backend v1.2.0
This commit is contained in:
William Oldham 2023-12-21 20:21:44 +00:00 committed by GitHub
parent f4e9a96666 d5851b522a
commit 26316c7755
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 147 additions and 283 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.docker/development/docker-compose.yml
View File

@ -12,6 +12,8 @@ services:
POSTGRES_PASSWORD: postgres
volumes:
- 'postgres_data:/var/lib/postgresql/data'
redis:
image: redis
# custom services
backend:

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "backend",
"version": "1.1.6",
"version": "1.2.0",
"private": true,
"homepage": "https://github.com/movie-web/backend",
"engines": {

4
src/config/fragments/docker.ts
View File

@ -4,4 +4,8 @@ export const dockerFragment: FragmentSchema = {
postgres: {
connection: 'postgres://postgres:postgres@postgres:5432/postgres',
},
ratelimits: {
enabled: true,
redisUrl: 'redis://redis:6379',
},
};

9
src/config/orm.ts
View File

@ -1,6 +1,13 @@
import { devFragment } from '@/config/fragments/dev';
import { dockerFragment } from '@/config/fragments/docker';
import { createConfigLoader } from 'neat-config';
import { z } from 'zod';
const fragments = {
dev: devFragment,
dockerdev: dockerFragment,
};
export const ormConfigSchema = z.object({
postgres: z.object({
// connection URL for postgres database
@ -15,6 +22,8 @@ export const ormConf = createConfigLoader()
prefix: 'MWB_',
})
.addFromFile('config.json')
.setFragmentKey('usePresets')
.addConfigFragments(fragments)
.addZodSchema(ormConfigSchema)
.freeze()
.load();

3
src/config/schema.ts
View File

@ -16,6 +16,9 @@ export const configSchema = z.object({
// should it trust reverse proxy headers? (for ip gathering)
trustProxy: z.coerce.boolean().default(false),
// should it trust cloudflare headers? (for ip gathering, cloudflare has priority)
trustCloudflare: z.coerce.boolean().default(false),
// prefix for where the instance is run on. for example set it to /backend if you're hosting it on example.com/backend
// if this is set, do not apply url rewriting before proxing
basePath: z.string().default('/'),

137
src/db/migrations/.snapshot-movie_web.json → src/db/migrations/.snapshot-postgres.json
View File

@ -268,143 +268,6 @@
"checks": [],
"foreignKeys": {}
},
{
"columns": {
"id": {
"name": "id",
"type": "uuid",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": false,
"mappedType": "uuid"
},
"tmdb_id": {
"name": "tmdb_id",
"type": "varchar(255)",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": false,
"mappedType": "string"
},
"type": {
"name": "type",
"type": "varchar(255)",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": false,
"mappedType": "string"
},
"title": {
"name": "title",
"type": "varchar(255)",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": false,
"mappedType": "string"
},
"season_id": {
"name": "season_id",
"type": "varchar(255)",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": true,
"mappedType": "string"
},
"episode_id": {
"name": "episode_id",
"type": "varchar(255)",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": true,
"mappedType": "string"
},
"created_at": {
"name": "created_at",
"type": "timestamptz(0)",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": false,
"length": 0,
"mappedType": "datetime"
},
"status": {
"name": "status",
"type": "varchar(255)",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": false,
"mappedType": "string"
},
"provider_id": {
"name": "provider_id",
"type": "varchar(255)",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": false,
"mappedType": "string"
},
"embed_id": {
"name": "embed_id",
"type": "varchar(255)",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": true,
"mappedType": "string"
},
"error_message": {
"name": "error_message",
"type": "text",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": true,
"mappedType": "text"
},
"full_error": {
"name": "full_error",
"type": "text",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": true,
"mappedType": "text"
},
"hostname": {
"name": "hostname",
"type": "varchar(255)",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": false,
"mappedType": "string"
}
},
"name": "provider_metrics",
"schema": "public",
"indexes": [
{
"keyName": "provider_metrics_pkey",
"columnNames": [
"id"
],
"composite": false,
"primary": true,
"unique": true
}
],
"checks": [],
"foreignKeys": {}
},
{
"columns": {
"id": {

13
src/db/migrations/Migration20231221185725.ts Normal file
View File

@ -0,0 +1,13 @@
import { Migration } from '@mikro-orm/migrations';
export class Migration20231221185725 extends Migration {
async up(): Promise<void> {
this.addSql('drop table if exists "provider_metrics" cascade;');
}
async down(): Promise<void> {
this.addSql('create table "provider_metrics" ("id" uuid not null default null, "tmdb_id" varchar not null default null, "type" varchar not null default null, "title" varchar not null default null, "season_id" varchar null default null, "episode_id" varchar null default null, "created_at" timestamptz not null default null, "status" varchar not null default null, "provider_id" varchar not null default null, "embed_id" varchar null default null, "error_message" text null default null, "full_error" text null default null, "hostname" varchar not null default null, constraint "provider_metrics_pkey" primary key ("id"));');
}
}

51
src/db/models/ProviderMetrics.ts
View File

@ -1,51 +0,0 @@
import { Entity, PrimaryKey, Property } from '@mikro-orm/core';
import { randomUUID } from 'crypto';
export const status = {
failed: 'failed',
notfound: 'notfound',
success: 'success',
} as const;
type Status = keyof typeof status;
@Entity({ tableName: 'provider_metrics' })
export class ProviderMetric {
@PrimaryKey({ name: 'id', type: 'uuid' })
id: string = randomUUID();
@Property({ name: 'tmdb_id' })
tmdbId!: string;
@Property({ name: 'type' })
type!: string;
@Property({ name: 'title' })
title!: string;
@Property({ name: 'season_id', nullable: true })
seasonId?: string;
@Property({ name: 'episode_id', nullable: true })
episodeId?: string;
@Property({ name: 'created_at', type: 'date' })
createdAt = new Date();
@Property({ name: 'status' })
status!: Status;
@Property({ name: 'provider_id' })
providerId!: string;
@Property({ name: 'embed_id', nullable: true })
embedId?: string;
@Property({ name: 'error_message', nullable: true, type: 'text' })
errorMessage?: string;
@Property({ name: 'full_error', nullable: true, type: 'text' })
fullError?: string;
@Property({ name: 'hostname' })
hostname!: string;
}

2
src/modules/jobs/index.ts
View File

@ -1,11 +1,9 @@
import { challengeCodeJob } from '@/modules/jobs/list/challengeCode';
import { sessionExpiryJob } from '@/modules/jobs/list/sessionExpiry';
import { userDeletionJob } from '@/modules/jobs/list/userDeletion';
import { providerMetricCleanupJob } from '@/modules/jobs/list/providerMetricCleanup';
export async function setupJobs() {
challengeCodeJob.start();
sessionExpiryJob.start();
userDeletionJob.start();
providerMetricCleanupJob.start();
}

27
src/modules/jobs/list/providerMetricCleanup.ts
View File

@ -1,27 +0,0 @@
import { ProviderMetric } from '@/db/models/ProviderMetrics';
import { job } from '@/modules/jobs/job';
import ms from 'ms';
// every day at 12:00:00
export const providerMetricCleanupJob = job(
'provider-metric-cleanup',
'0 12 * * *',
async ({ em, log }) => {
const now = new Date();
const thirtyDaysAgo = new Date(now.getTime() - ms('30d'));
const deletedMetrics = await em
.createQueryBuilder(ProviderMetric)
.delete()
.where({
createdAt: {
$lt: thirtyDaysAgo,
},
})
.execute<{ affectedRows: number }>('run');
log.info(
`Removed ${deletedMetrics.affectedRows} metrics that were older than 30 days`,
);
},
);

56
src/modules/metrics/index.ts
View File

@ -9,17 +9,10 @@ const log = scopedLogger('metrics');
export type Metrics = {
user: Counter<'namespace'>;
providerMetrics: Counter<
| 'title'
| 'tmdb_id'
| 'season_id'
| 'episode_id'
| 'status'
| 'type'
| 'provider_id'
| 'embed_id'
| 'hostname'
>;
captchaSolves: Counter<'success'>;
providerHostnames: Counter<'hostname'>;
providerStatuses: Counter<'provider_id' | 'status'>;
watchMetrics: Counter<'title' | 'tmdb_full_id' | 'provider_id' | 'success'>;
};
let metrics: null | Metrics = null;
@ -42,31 +35,38 @@ export async function setupMetrics(app: FastifyInstance) {
metrics = {
user: new Counter({
name: 'user_count',
help: 'user_help',
name: 'mw_user_count',
help: 'mw_user_help',
labelNames: ['namespace'],
}),
providerMetrics: new Counter({
name: 'provider_metrics',
help: 'provider_metrics',
labelNames: [
'episode_id',
'provider_id',
'season_id',
'status',
'title',
'tmdb_id',
'type',
'embed_id',
'hostname',
],
captchaSolves: new Counter({
name: 'mw_captcha_solves',
help: 'mw_captcha_solves',
labelNames: ['success'],
}),
providerHostnames: new Counter({
name: 'mw_provider_hostname_count',
help: 'mw_provider_hostname_count',
labelNames: ['hostname'],
}),
providerStatuses: new Counter({
name: 'mw_provider_status_count',
help: 'mw_provider_status_count',
labelNames: ['provider_id', 'status'],
}),
watchMetrics: new Counter({
name: 'mw_media_watch_count',
help: 'mw_media_watch_count',
labelNames: ['title', 'tmdb_full_id', 'provider_id', 'success'],
}),
};
const promClient = app.metrics.client;
promClient.register.registerMetric(metrics.user);
promClient.register.registerMetric(metrics.providerMetrics);
promClient.register.registerMetric(metrics.providerHostnames);
promClient.register.registerMetric(metrics.providerStatuses);
promClient.register.registerMetric(metrics.watchMetrics);
const orm = getORM();
const em = orm.em.fork();

7
src/modules/ratelimits/limiter.ts
View File

@ -2,6 +2,7 @@ import Redis from 'ioredis';
import RateLimiter from 'async-ratelimiter';
import ms from 'ms';
import { StatusError } from '@/services/error';
import { IpReq, getIp } from '@/services/ip';
export interface LimiterOptions {
redis: Redis;
@ -26,8 +27,8 @@ export class Limiter {
this.redis = ops.redis;
}
async bump(req: { ip: string }, ops: BucketOptions) {
const ip = req.ip;
async bump(req: IpReq, ops: BucketOptions) {
const ip = getIp(req);
if (!this.buckets[ops.id]) {
this.buckets[ops.id] = {
limiter: new RateLimiter({
@ -54,7 +55,7 @@ export class Limiter {
};
}
async assertAndBump(req: { ip: string }, ops: BucketOptions) {
async assertAndBump(req: IpReq, ops: BucketOptions) {
const { hasBeenLimited } = await this.bump(req, ops);
if (hasBeenLimited) {
throw new StatusError('Ratelimited', 429);

84
src/routes/metrics.ts
View File

@ -1,8 +1,8 @@
import { handle } from '@/services/handler';
import { makeRouter } from '@/services/router';
import { z } from 'zod';
import { ProviderMetric, status } from '@/db/models/ProviderMetrics';
import { getMetrics } from '@/modules/metrics';
import { status } from '@/routes/statuses';
const metricsProviderSchema = z.object({
tmdbId: z.string(),
@ -29,7 +29,7 @@ export const metricsRouter = makeRouter((app) => {
body: metricsProviderInputSchema,
},
},
handle(async ({ em, body, req, limiter }) => {
handle(async ({ body, req, limiter }) => {
await limiter?.assertAndBump(req, {
id: 'provider_metrics',
max: 300,
@ -37,43 +37,59 @@ export const metricsRouter = makeRouter((app) => {
window: '30m',
});
const hostname = req.headers.origin?.slice(0, 255) ?? 'unknown origin';
const entities = body.items.map((v) => {
const errorMessage = v.errorMessage?.slice(0, 200);
const truncatedFullError = v.fullError?.slice(0, 2000);
const metric = new ProviderMetric();
em.assign(metric, {
providerId: v.providerId,
embedId: v.embedId,
fullError: truncatedFullError,
errorMessage: errorMessage,
episodeId: v.episodeId,
seasonId: v.seasonId,
status: v.status,
title: v.title,
tmdbId: v.tmdbId,
type: v.type,
hostname,
});
return metric;
const hostname = req.headers.origin?.slice(0, 255) ?? '<UNKNOWN>';
getMetrics().providerHostnames.inc({
hostname,
});
entities.forEach((entity) => {
getMetrics().providerMetrics.inc({
episode_id: entity.episodeId,
provider_id: entity.providerId,
season_id: entity.seasonId,
status: entity.status,
title: entity.title,
tmdb_id: entity.tmdbId,
type: entity.type,
hostname,
body.items.forEach((item) => {
getMetrics().providerStatuses.inc({
provider_id: item.embedId ?? item.providerId,
status: item.status,
});
});
await em.persistAndFlush(entities);
const itemList = [...body.items];
itemList.reverse();
const lastSuccessfulItem = body.items.find(
(v) => v.status === status.success,
);
const lastItem = itemList[0];
if (lastItem) {
getMetrics().watchMetrics.inc({
tmdb_full_id: lastItem.type + '-' + lastItem.tmdbId,
provider_id: lastSuccessfulItem?.providerId ?? lastItem.providerId,
title: lastItem.title,
success: (!!lastSuccessfulItem).toString(),
});
}
return true;
}),
);
app.post(
'/metrics/captcha',
{
schema: {
body: z.object({
success: z.boolean(),
}),
},
},
handle(async ({ body, req, limiter }) => {
await limiter?.assertAndBump(req, {
id: 'captcha_solves',
max: 300,
inc: 1,
window: '30m',
});
getMetrics().captchaSolves.inc({
success: body.success.toString(),
});
return true;
}),
);

6
src/routes/statuses.ts Normal file
View File

@ -0,0 +1,6 @@
export const status = {
failed: 'failed',
notfound: 'notfound',
success: 'success',
} as const;
export type Status = keyof typeof status;

27
src/services/ip.ts Normal file
View File

@ -0,0 +1,27 @@
import { conf } from '@/config';
import { IncomingHttpHeaders } from 'http';
export type IpReq = {
ip: string;
headers: IncomingHttpHeaders;
};
const trustCloudflare = conf.server.trustCloudflare;
function getSingleHeader(
headers: IncomingHttpHeaders,
key: string,
): string | undefined {
const header = headers[key];
if (Array.isArray(header)) return header[0];
return header;
}
export function getIp(req: IpReq) {
const cfIp = getSingleHeader(req.headers, 'cf-connecting-ip');
if (trustCloudflare && cfIp) {
return cfIp;
}
return req.ip;
}