dev
/
backend
mirror of https://github.com/sussy-code/backend.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add rate limits 

Co-authored-by: mrjvs <mistrjvs@gmail.com>
This commit is contained in:
William Oldham 2023-11-04 14:52:11 +00:00
parent 616778ab6d
commit 78b4dbd705
12 changed files with 216 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -32,7 +32,7 @@ Backend for movie-web
- [x] provider metrics
- [x] cleanup old metrics in DB
- [x] endpoint to consume and store metrics
- [ ] pass metrics to prometheus
- [x] pass metrics to prometheus
## Second todo list
- [X] think of privacy centric method of auth

4
package.json
View File

@ -38,10 +38,12 @@
"@mikro-orm/core": "^5.9.0",
"@mikro-orm/postgresql": "^5.9.0",
"@types/ms": "^0.7.33",
"async-ratelimiter": "^1.3.12",
"cron": "^3.1.5",
"fastify": "^4.21.0",
"fastify-metrics": "^10.3.2",
"fastify-metrics": "^10.3.3",
"fastify-type-provider-zod": "^1.1.9",
"ioredis": "^5.3.2",
"jsonwebtoken": "^9.0.2",
"ms": "^2.1.3",
"nanoid": "^3.3.6",

74
pnpm-lock.yaml
View File

@ -17,6 +17,9 @@ dependencies:
'@types/ms':
specifier: ^0.7.33
version: 0.7.33
async-ratelimiter:
specifier: ^1.3.12
version: 1.3.12
cron:
specifier: ^3.1.5
version: 3.1.5
@ -24,11 +27,14 @@ dependencies:
specifier: ^4.21.0
version: 4.24.3
fastify-metrics:
specifier: ^10.3.2
version: 10.3.2(fastify@4.24.3)
specifier: ^10.3.3
version: 10.3.3(fastify@4.24.3)
fastify-type-provider-zod:
specifier: ^1.1.9
version: 1.1.9(fastify@4.24.3)(zod@3.22.4)
ioredis:
specifier: ^5.3.2
version: 5.3.2
jsonwebtoken:
specifier: ^9.0.2
version: 9.0.2
@ -230,6 +236,10 @@ packages:
resolution: {integrity: sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==}
dev: true
/@ioredis/commands@1.2.0:
resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==}
dev: false
/@isaacs/cliui@8.0.2:
resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
engines: {node: '>=12'}
@ -741,6 +751,11 @@ packages:
resolution: {integrity: sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==}
engines: {node: '>=8'}
/async-ratelimiter@1.3.12:
resolution: {integrity: sha512-W7WWxWMjJ+XEZCyQhEWGrskqDgz3k2UWM/aUlatSl3ejFLwpM/G90AYSgkHHXeY2S53fiP204GITnmIxrJMsSQ==}
engines: {node: '>= 8'}
dev: false
/async@3.2.4:
resolution: {integrity: sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==}
dev: false
@ -859,6 +874,11 @@ packages:
fsevents: 2.3.3
dev: true
/cluster-key-slot@1.1.2:
resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==}
engines: {node: '>=0.10.0'}
dev: false
/color-convert@1.9.3:
resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==}
dependencies:
@ -998,6 +1018,11 @@ packages:
engines: {node: '>=12'}
dev: true
/denque@2.1.0:
resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==}
engines: {node: '>=0.10'}
dev: false
/diff@4.0.2:
resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==}
engines: {node: '>=0.3.1'}
@ -1280,8 +1305,8 @@ packages:
resolution: {integrity: sha512-cIusKBIt/R/oI6z/1nyfe2FvGKVTohVRfvkOhvx0nCEW+xf5NoCXjAHcWp93uOUBchzYcsvPlrapAdX1uW+YGg==}
dev: false
/fastify-metrics@10.3.2(fastify@4.24.3):
resolution: {integrity: sha512-02SEIGH02zfguqRMho0LB8L7YVAj5cIgWM0iqZslIErqaUWc1iHVAOC+YXYG3S2DZU6VHdFaMyuxjEOCQHAETA==}
/fastify-metrics@10.3.3(fastify@4.24.3):
resolution: {integrity: sha512-TmMcfrMWBSbA7yk31tFtJnWKtNXLSO7jmTRIjPX9HKC4pLmyd0JnOQ3r9XCYnev6NL9/eVRXxNfrsqQdKTLZkw==}
peerDependencies:
fastify: '>=4'
dependencies:
@ -1565,6 +1590,23 @@ packages:
engines: {node: '>= 0.10'}
dev: false
/ioredis@5.3.2:
resolution: {integrity: sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==}
engines: {node: '>=12.22.0'}
dependencies:
'@ioredis/commands': 1.2.0
cluster-key-slot: 1.1.2
debug: 4.3.4
denque: 2.1.0
lodash.defaults: 4.2.0
lodash.isarguments: 3.1.0
redis-errors: 1.2.0
redis-parser: 3.0.0
standard-as-callback: 2.1.0
transitivePeerDependencies:
- supports-color
dev: false
/ipaddr.js@1.9.1:
resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==}
engines: {node: '>= 0.10'}
@ -1818,10 +1860,18 @@ packages:
p-locate: 5.0.0
dev: true
/lodash.defaults@4.2.0:
resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==}
dev: false
/lodash.includes@4.3.0:
resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==}
dev: false
/lodash.isarguments@3.1.0:
resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==}
dev: false
/lodash.isboolean@3.0.3:
resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==}
dev: false
@ -2380,6 +2430,18 @@ packages:
resolve: 1.22.8
dev: false
/redis-errors@1.2.0:
resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==}
engines: {node: '>=4'}
dev: false
/redis-parser@3.0.0:
resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==}
engines: {node: '>=4'}
dependencies:
redis-errors: 1.2.0
dev: false
/reflect-metadata@0.1.13:
resolution: {integrity: sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==}
dev: false
@ -2536,6 +2598,10 @@ packages:
resolution: {integrity: sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==}
dev: false
/standard-as-callback@2.1.0:
resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==}
dev: false
/string-width@4.2.3:
resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==}
engines: {node: '>=8'}

7
src/config/schema.ts
View File

@ -59,4 +59,11 @@ export const configSchema = z.object({
clientKey: z.string().min(1).optional(),
})
.default({}),
ratelimits: z
.object({
// enabled captchas on register
enabled: z.coerce.boolean().default(false),
redisUrl: z.string().optional(),
})
.default({}),
});

2
src/main.ts
View File

@ -2,6 +2,7 @@ import { setupFastify, startFastify } from '@/modules/fastify';
import { setupJobs } from '@/modules/jobs';
import { setupMetrics } from '@/modules/metrics';
import { setupMikroORM } from '@/modules/mikro';
import { setupRatelimits } from '@/modules/ratelimits';
import { scopedLogger } from '@/services/logger';
const log = scopedLogger('mw-backend');
@ -11,6 +12,7 @@ async function bootstrap(): Promise<void> {
evt: 'setup',
});
await setupRatelimits();
const app = await setupFastify();
await setupMikroORM();
await setupMetrics(app);

24
src/modules/ratelimits/index.ts Normal file
View File

@ -0,0 +1,24 @@
import { conf } from '@/config';
import { Limiter } from '@/modules/ratelimits/limiter';
import { connectRedis } from '@/modules/ratelimits/redis';
import { scopedLogger } from '@/services/logger';
const log = scopedLogger('ratelimits');
let limiter: null | Limiter = null;
export function getLimiter() {
return limiter;
}
export async function setupRatelimits() {
if (!conf.ratelimits.enabled) {
log.warn('Ratelimits disabled!');
return;
}
const redis = await connectRedis();
limiter = new Limiter({
redis,
});
log.info('Ratelimits have been setup!');
}

63
src/modules/ratelimits/limiter.ts Normal file
View File

@ -0,0 +1,63 @@
import Redis from 'ioredis';
import RateLimiter from 'async-ratelimiter';
import ms from 'ms';
import { StatusError } from '@/services/error';
export interface LimiterOptions {
redis: Redis;
}
interface LimitBucket {
limiter: RateLimiter;
}
interface BucketOptions {
id: string;
window: string;
max: number;
inc?: number;
}
export class Limiter {
private redis: Redis;
private buckets: Record<string, LimitBucket> = {};
constructor(ops: LimiterOptions) {
this.redis = ops.redis;
}
async bump(req: { ip: string }, ops: BucketOptions) {
const ip = req.ip;
if (!this.buckets[ops.id]) {
this.buckets[ops.id] = {
limiter: new RateLimiter({
db: this.redis,
namespace: `RATELIMIT_${ops.id}`,
duration: ms(ops.window),
max: ops.max,
}),
};
}
for (let i = 1; i < (ops.inc ?? 0); i++) {
await this.buckets[ops.id].limiter.get({
id: ip,
});
}
const currentLimit = await this.buckets[ops.id].limiter.get({
id: ip,
});
return {
hasBeenLimited: currentLimit.remaining <= 0,
limit: currentLimit,
};
}
async assertAndBump(req: { ip: string }, ops: BucketOptions) {
const { hasBeenLimited } = await this.bump(req, ops);
if (hasBeenLimited) {
throw new StatusError('Ratelimited', 429);
}
}
}

7
src/modules/ratelimits/redis.ts Normal file
View File

@ -0,0 +1,7 @@
import { conf } from '@/config';
import Redis from 'ioredis';
export function connectRedis() {
if (!conf.ratelimits.redisUrl) throw new Error('missing redis URL');
return new Redis(conf.ratelimits.redisUrl);
}

16
src/routes/auth/login.ts
View File

@ -25,7 +25,13 @@ export const loginAuthRouter = makeRouter((app) => {
app.post(
'/auth/login/start',
{ schema: { body: startSchema } },
handle(async ({ em, body }) => {
handle(async ({ em, body, limiter, req }) => {
await limiter?.assertAndBump(req, {
id: 'login_challenge_tokens',
max: 20,
window: '10m',
});
const user = await em.findOne(User, { publicKey: body.publicKey });
if (user == null) {
@ -46,7 +52,13 @@ export const loginAuthRouter = makeRouter((app) => {
app.post(
'/auth/login/complete',
{ schema: { body: completeSchema } },
handle(async ({ em, body, req }) => {
handle(async ({ em, body, req, limiter }) => {
await limiter?.assertAndBump(req, {
id: 'login_complete',
max: 20,
window: '10m',
});
await assertChallengeCode(
em,
body.challenge.code,

15
src/routes/auth/manage.ts
View File

@ -32,7 +32,12 @@ export const manageAuthRouter = makeRouter((app) => {
app.post(
'/auth/register/start',
{ schema: { body: startSchema } },
handle(async ({ em, body }) => {
handle(async ({ em, body, limiter, req }) => {
await limiter?.assertAndBump(req, {
id: 'register_challenge_tokens',
max: 10,
window: '10m',
});
await assertCaptcha(body.captchaToken);
const challenge = new ChallengeCode();
@ -50,7 +55,13 @@ export const manageAuthRouter = makeRouter((app) => {
app.post(
'/auth/register/complete',
{ schema: { body: completeSchema } },
handle(async ({ em, body, req }) => {
handle(async ({ em, body, req, limiter }) => {
await limiter?.assertAndBump(req, {
id: 'register_complete',
max: 10,
window: '10m',
});
await assertChallengeCode(
em,
body.challenge.code,

9
src/routes/metrics.ts
View File

@ -29,7 +29,14 @@ export const metricsRouter = makeRouter((app) => {
body: metricsProviderInputSchema,
},
},
handle(async ({ em, body }) => {
handle(async ({ em, body, req, limiter }) => {
await limiter?.assertAndBump(req, {
id: 'provider_metrics',
max: 300,
inc: body.items.length,
window: '30m',
});
const entities = body.items.map((v) => {
const metric = new ProviderMetric();
em.assign(metric, {

4
src/services/handler.ts
View File

@ -1,4 +1,6 @@
import { getORM } from '@/modules/mikro';
import { getLimiter } from '@/modules/ratelimits';
import { Limiter } from '@/modules/ratelimits/limiter';
import { makeAuthContext } from '@/services/auth';
import { EntityManager } from '@mikro-orm/postgresql';
import {
@ -74,6 +76,7 @@ export type RequestContext<
Logger
>['query'];
em: EntityManager;
limiter: Limiter | null;
auth: ReturnType<typeof makeAuthContext>;
};
@ -124,6 +127,7 @@ export function handle<
query: req.query,
em,
auth: makeAuthContext(em, req),
limiter: getLimiter(),
}),
);
};