Add rate limits
Co-authored-by: mrjvs <mistrjvs@gmail.com>
This commit is contained in:
parent
616778ab6d
commit
78b4dbd705
|
@ -32,7 +32,7 @@ Backend for movie-web
|
|||
- [x] provider metrics
|
||||
- [x] cleanup old metrics in DB
|
||||
- [x] endpoint to consume and store metrics
|
||||
- [ ] pass metrics to prometheus
|
||||
- [x] pass metrics to prometheus
|
||||
|
||||
## Second todo list
|
||||
- [X] think of privacy centric method of auth
|
||||
|
|
|
@ -38,10 +38,12 @@
|
|||
"@mikro-orm/core": "^5.9.0",
|
||||
"@mikro-orm/postgresql": "^5.9.0",
|
||||
"@types/ms": "^0.7.33",
|
||||
"async-ratelimiter": "^1.3.12",
|
||||
"cron": "^3.1.5",
|
||||
"fastify": "^4.21.0",
|
||||
"fastify-metrics": "^10.3.2",
|
||||
"fastify-metrics": "^10.3.3",
|
||||
"fastify-type-provider-zod": "^1.1.9",
|
||||
"ioredis": "^5.3.2",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"ms": "^2.1.3",
|
||||
"nanoid": "^3.3.6",
|
||||
|
|
|
@ -17,6 +17,9 @@ dependencies:
|
|||
'@types/ms':
|
||||
specifier: ^0.7.33
|
||||
version: 0.7.33
|
||||
async-ratelimiter:
|
||||
specifier: ^1.3.12
|
||||
version: 1.3.12
|
||||
cron:
|
||||
specifier: ^3.1.5
|
||||
version: 3.1.5
|
||||
|
@ -24,11 +27,14 @@ dependencies:
|
|||
specifier: ^4.21.0
|
||||
version: 4.24.3
|
||||
fastify-metrics:
|
||||
specifier: ^10.3.2
|
||||
version: 10.3.2(fastify@4.24.3)
|
||||
specifier: ^10.3.3
|
||||
version: 10.3.3(fastify@4.24.3)
|
||||
fastify-type-provider-zod:
|
||||
specifier: ^1.1.9
|
||||
version: 1.1.9(fastify@4.24.3)(zod@3.22.4)
|
||||
ioredis:
|
||||
specifier: ^5.3.2
|
||||
version: 5.3.2
|
||||
jsonwebtoken:
|
||||
specifier: ^9.0.2
|
||||
version: 9.0.2
|
||||
|
@ -230,6 +236,10 @@ packages:
|
|||
resolution: {integrity: sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw==}
|
||||
dev: true
|
||||
|
||||
/@ioredis/commands@1.2.0:
|
||||
resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==}
|
||||
dev: false
|
||||
|
||||
/@isaacs/cliui@8.0.2:
|
||||
resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==}
|
||||
engines: {node: '>=12'}
|
||||
|
@ -741,6 +751,11 @@ packages:
|
|||
resolution: {integrity: sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==}
|
||||
engines: {node: '>=8'}
|
||||
|
||||
/async-ratelimiter@1.3.12:
|
||||
resolution: {integrity: sha512-W7WWxWMjJ+XEZCyQhEWGrskqDgz3k2UWM/aUlatSl3ejFLwpM/G90AYSgkHHXeY2S53fiP204GITnmIxrJMsSQ==}
|
||||
engines: {node: '>= 8'}
|
||||
dev: false
|
||||
|
||||
/async@3.2.4:
|
||||
resolution: {integrity: sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==}
|
||||
dev: false
|
||||
|
@ -859,6 +874,11 @@ packages:
|
|||
fsevents: 2.3.3
|
||||
dev: true
|
||||
|
||||
/cluster-key-slot@1.1.2:
|
||||
resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==}
|
||||
engines: {node: '>=0.10.0'}
|
||||
dev: false
|
||||
|
||||
/color-convert@1.9.3:
|
||||
resolution: {integrity: sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==}
|
||||
dependencies:
|
||||
|
@ -998,6 +1018,11 @@ packages:
|
|||
engines: {node: '>=12'}
|
||||
dev: true
|
||||
|
||||
/denque@2.1.0:
|
||||
resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==}
|
||||
engines: {node: '>=0.10'}
|
||||
dev: false
|
||||
|
||||
/diff@4.0.2:
|
||||
resolution: {integrity: sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==}
|
||||
engines: {node: '>=0.3.1'}
|
||||
|
@ -1280,8 +1305,8 @@ packages:
|
|||
resolution: {integrity: sha512-cIusKBIt/R/oI6z/1nyfe2FvGKVTohVRfvkOhvx0nCEW+xf5NoCXjAHcWp93uOUBchzYcsvPlrapAdX1uW+YGg==}
|
||||
dev: false
|
||||
|
||||
/fastify-metrics@10.3.2(fastify@4.24.3):
|
||||
resolution: {integrity: sha512-02SEIGH02zfguqRMho0LB8L7YVAj5cIgWM0iqZslIErqaUWc1iHVAOC+YXYG3S2DZU6VHdFaMyuxjEOCQHAETA==}
|
||||
/fastify-metrics@10.3.3(fastify@4.24.3):
|
||||
resolution: {integrity: sha512-TmMcfrMWBSbA7yk31tFtJnWKtNXLSO7jmTRIjPX9HKC4pLmyd0JnOQ3r9XCYnev6NL9/eVRXxNfrsqQdKTLZkw==}
|
||||
peerDependencies:
|
||||
fastify: '>=4'
|
||||
dependencies:
|
||||
|
@ -1565,6 +1590,23 @@ packages:
|
|||
engines: {node: '>= 0.10'}
|
||||
dev: false
|
||||
|
||||
/ioredis@5.3.2:
|
||||
resolution: {integrity: sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA==}
|
||||
engines: {node: '>=12.22.0'}
|
||||
dependencies:
|
||||
'@ioredis/commands': 1.2.0
|
||||
cluster-key-slot: 1.1.2
|
||||
debug: 4.3.4
|
||||
denque: 2.1.0
|
||||
lodash.defaults: 4.2.0
|
||||
lodash.isarguments: 3.1.0
|
||||
redis-errors: 1.2.0
|
||||
redis-parser: 3.0.0
|
||||
standard-as-callback: 2.1.0
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
dev: false
|
||||
|
||||
/ipaddr.js@1.9.1:
|
||||
resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==}
|
||||
engines: {node: '>= 0.10'}
|
||||
|
@ -1818,10 +1860,18 @@ packages:
|
|||
p-locate: 5.0.0
|
||||
dev: true
|
||||
|
||||
/lodash.defaults@4.2.0:
|
||||
resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==}
|
||||
dev: false
|
||||
|
||||
/lodash.includes@4.3.0:
|
||||
resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==}
|
||||
dev: false
|
||||
|
||||
/lodash.isarguments@3.1.0:
|
||||
resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==}
|
||||
dev: false
|
||||
|
||||
/lodash.isboolean@3.0.3:
|
||||
resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==}
|
||||
dev: false
|
||||
|
@ -2380,6 +2430,18 @@ packages:
|
|||
resolve: 1.22.8
|
||||
dev: false
|
||||
|
||||
/redis-errors@1.2.0:
|
||||
resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==}
|
||||
engines: {node: '>=4'}
|
||||
dev: false
|
||||
|
||||
/redis-parser@3.0.0:
|
||||
resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==}
|
||||
engines: {node: '>=4'}
|
||||
dependencies:
|
||||
redis-errors: 1.2.0
|
||||
dev: false
|
||||
|
||||
/reflect-metadata@0.1.13:
|
||||
resolution: {integrity: sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==}
|
||||
dev: false
|
||||
|
@ -2536,6 +2598,10 @@ packages:
|
|||
resolution: {integrity: sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==}
|
||||
dev: false
|
||||
|
||||
/standard-as-callback@2.1.0:
|
||||
resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==}
|
||||
dev: false
|
||||
|
||||
/string-width@4.2.3:
|
||||
resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==}
|
||||
engines: {node: '>=8'}
|
||||
|
|
|
@ -59,4 +59,11 @@ export const configSchema = z.object({
|
|||
clientKey: z.string().min(1).optional(),
|
||||
})
|
||||
.default({}),
|
||||
ratelimits: z
|
||||
.object({
|
||||
// enabled captchas on register
|
||||
enabled: z.coerce.boolean().default(false),
|
||||
redisUrl: z.string().optional(),
|
||||
})
|
||||
.default({}),
|
||||
});
|
||||
|
|
|
@ -2,6 +2,7 @@ import { setupFastify, startFastify } from '@/modules/fastify';
|
|||
import { setupJobs } from '@/modules/jobs';
|
||||
import { setupMetrics } from '@/modules/metrics';
|
||||
import { setupMikroORM } from '@/modules/mikro';
|
||||
import { setupRatelimits } from '@/modules/ratelimits';
|
||||
import { scopedLogger } from '@/services/logger';
|
||||
|
||||
const log = scopedLogger('mw-backend');
|
||||
|
@ -11,6 +12,7 @@ async function bootstrap(): Promise<void> {
|
|||
evt: 'setup',
|
||||
});
|
||||
|
||||
await setupRatelimits();
|
||||
const app = await setupFastify();
|
||||
await setupMikroORM();
|
||||
await setupMetrics(app);
|
||||
|
|
|
@ -0,0 +1,24 @@
|
|||
import { conf } from '@/config';
|
||||
import { Limiter } from '@/modules/ratelimits/limiter';
|
||||
import { connectRedis } from '@/modules/ratelimits/redis';
|
||||
import { scopedLogger } from '@/services/logger';
|
||||
|
||||
const log = scopedLogger('ratelimits');
|
||||
|
||||
let limiter: null | Limiter = null;
|
||||
|
||||
export function getLimiter() {
|
||||
return limiter;
|
||||
}
|
||||
|
||||
export async function setupRatelimits() {
|
||||
if (!conf.ratelimits.enabled) {
|
||||
log.warn('Ratelimits disabled!');
|
||||
return;
|
||||
}
|
||||
const redis = await connectRedis();
|
||||
limiter = new Limiter({
|
||||
redis,
|
||||
});
|
||||
log.info('Ratelimits have been setup!');
|
||||
}
|
|
@ -0,0 +1,63 @@
|
|||
import Redis from 'ioredis';
|
||||
import RateLimiter from 'async-ratelimiter';
|
||||
import ms from 'ms';
|
||||
import { StatusError } from '@/services/error';
|
||||
|
||||
export interface LimiterOptions {
|
||||
redis: Redis;
|
||||
}
|
||||
|
||||
interface LimitBucket {
|
||||
limiter: RateLimiter;
|
||||
}
|
||||
|
||||
interface BucketOptions {
|
||||
id: string;
|
||||
window: string;
|
||||
max: number;
|
||||
inc?: number;
|
||||
}
|
||||
|
||||
export class Limiter {
|
||||
private redis: Redis;
|
||||
private buckets: Record<string, LimitBucket> = {};
|
||||
|
||||
constructor(ops: LimiterOptions) {
|
||||
this.redis = ops.redis;
|
||||
}
|
||||
|
||||
async bump(req: { ip: string }, ops: BucketOptions) {
|
||||
const ip = req.ip;
|
||||
if (!this.buckets[ops.id]) {
|
||||
this.buckets[ops.id] = {
|
||||
limiter: new RateLimiter({
|
||||
db: this.redis,
|
||||
namespace: `RATELIMIT_${ops.id}`,
|
||||
duration: ms(ops.window),
|
||||
max: ops.max,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
for (let i = 1; i < (ops.inc ?? 0); i++) {
|
||||
await this.buckets[ops.id].limiter.get({
|
||||
id: ip,
|
||||
});
|
||||
}
|
||||
const currentLimit = await this.buckets[ops.id].limiter.get({
|
||||
id: ip,
|
||||
});
|
||||
|
||||
return {
|
||||
hasBeenLimited: currentLimit.remaining <= 0,
|
||||
limit: currentLimit,
|
||||
};
|
||||
}
|
||||
|
||||
async assertAndBump(req: { ip: string }, ops: BucketOptions) {
|
||||
const { hasBeenLimited } = await this.bump(req, ops);
|
||||
if (hasBeenLimited) {
|
||||
throw new StatusError('Ratelimited', 429);
|
||||
}
|
||||
}
|
||||
}
|
|
@ -0,0 +1,7 @@
|
|||
import { conf } from '@/config';
|
||||
import Redis from 'ioredis';
|
||||
|
||||
export function connectRedis() {
|
||||
if (!conf.ratelimits.redisUrl) throw new Error('missing redis URL');
|
||||
return new Redis(conf.ratelimits.redisUrl);
|
||||
}
|
|
@ -25,7 +25,13 @@ export const loginAuthRouter = makeRouter((app) => {
|
|||
app.post(
|
||||
'/auth/login/start',
|
||||
{ schema: { body: startSchema } },
|
||||
handle(async ({ em, body }) => {
|
||||
handle(async ({ em, body, limiter, req }) => {
|
||||
await limiter?.assertAndBump(req, {
|
||||
id: 'login_challenge_tokens',
|
||||
max: 20,
|
||||
window: '10m',
|
||||
});
|
||||
|
||||
const user = await em.findOne(User, { publicKey: body.publicKey });
|
||||
|
||||
if (user == null) {
|
||||
|
@ -46,7 +52,13 @@ export const loginAuthRouter = makeRouter((app) => {
|
|||
app.post(
|
||||
'/auth/login/complete',
|
||||
{ schema: { body: completeSchema } },
|
||||
handle(async ({ em, body, req }) => {
|
||||
handle(async ({ em, body, req, limiter }) => {
|
||||
await limiter?.assertAndBump(req, {
|
||||
id: 'login_complete',
|
||||
max: 20,
|
||||
window: '10m',
|
||||
});
|
||||
|
||||
await assertChallengeCode(
|
||||
em,
|
||||
body.challenge.code,
|
||||
|
|
|
@ -32,7 +32,12 @@ export const manageAuthRouter = makeRouter((app) => {
|
|||
app.post(
|
||||
'/auth/register/start',
|
||||
{ schema: { body: startSchema } },
|
||||
handle(async ({ em, body }) => {
|
||||
handle(async ({ em, body, limiter, req }) => {
|
||||
await limiter?.assertAndBump(req, {
|
||||
id: 'register_challenge_tokens',
|
||||
max: 10,
|
||||
window: '10m',
|
||||
});
|
||||
await assertCaptcha(body.captchaToken);
|
||||
|
||||
const challenge = new ChallengeCode();
|
||||
|
@ -50,7 +55,13 @@ export const manageAuthRouter = makeRouter((app) => {
|
|||
app.post(
|
||||
'/auth/register/complete',
|
||||
{ schema: { body: completeSchema } },
|
||||
handle(async ({ em, body, req }) => {
|
||||
handle(async ({ em, body, req, limiter }) => {
|
||||
await limiter?.assertAndBump(req, {
|
||||
id: 'register_complete',
|
||||
max: 10,
|
||||
window: '10m',
|
||||
});
|
||||
|
||||
await assertChallengeCode(
|
||||
em,
|
||||
body.challenge.code,
|
||||
|
|
|
@ -29,7 +29,14 @@ export const metricsRouter = makeRouter((app) => {
|
|||
body: metricsProviderInputSchema,
|
||||
},
|
||||
},
|
||||
handle(async ({ em, body }) => {
|
||||
handle(async ({ em, body, req, limiter }) => {
|
||||
await limiter?.assertAndBump(req, {
|
||||
id: 'provider_metrics',
|
||||
max: 300,
|
||||
inc: body.items.length,
|
||||
window: '30m',
|
||||
});
|
||||
|
||||
const entities = body.items.map((v) => {
|
||||
const metric = new ProviderMetric();
|
||||
em.assign(metric, {
|
||||
|
|
|
@ -1,4 +1,6 @@
|
|||
import { getORM } from '@/modules/mikro';
|
||||
import { getLimiter } from '@/modules/ratelimits';
|
||||
import { Limiter } from '@/modules/ratelimits/limiter';
|
||||
import { makeAuthContext } from '@/services/auth';
|
||||
import { EntityManager } from '@mikro-orm/postgresql';
|
||||
import {
|
||||
|
@ -74,6 +76,7 @@ export type RequestContext<
|
|||
Logger
|
||||
>['query'];
|
||||
em: EntityManager;
|
||||
limiter: Limiter | null;
|
||||
auth: ReturnType<typeof makeAuthContext>;
|
||||
};
|
||||
|
||||
|
@ -124,6 +127,7 @@ export function handle<
|
|||
query: req.query,
|
||||
em,
|
||||
auth: makeAuthContext(em, req),
|
||||
limiter: getLimiter(),
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
|
Loading…
Reference in New Issue