Merge pull request #12 from ztpn/proxy

Merging this for now, won't enable it on the frontend till the proxy is ready
This commit is contained in:
TPN 2024-07-16 10:47:02 +05:30 committed by GitHub
commit c8f6687b6b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 97 additions and 21 deletions

View File

@ -14,6 +14,7 @@ export interface ProviderControlsInput {
features: FeatureMap;
sources: Sourcerer[];
embeds: Embed[];
proxyStreams?: boolean; // temporary
}
export interface RunnerOptions {
@ -85,6 +86,7 @@ export function makeControls(ops: ProviderControlsInput): ProviderControls {
features: ops.features,
fetcher: makeFetcher(ops.fetcher),
proxiedFetcher: makeFetcher(ops.proxiedFetcher ?? ops.fetcher),
proxyStreams: ops.proxyStreams,
};
return {

View File

@ -18,10 +18,17 @@ export interface ProviderMakerOptions {
// Set this to true, if the requests will have the same IP as
// the device that the stream will be played on
consistentIpForRequests?: boolean;
// This is temporary
proxyStreams?: boolean;
}
export function makeProviders(ops: ProviderMakerOptions) {
const features = getTargetFeatures(ops.target, ops.consistentIpForRequests ?? false);
const features = getTargetFeatures(
ops.proxyStreams ? 'any' : ops.target,
ops.consistentIpForRequests ?? false,
ops.proxyStreams,
);
const list = getProviders(features, {
embeds: getBuiltinEmbeds(),
sources: getBuiltinSources(),
@ -33,5 +40,6 @@ export function makeProviders(ops: ProviderMakerOptions) {
features,
fetcher: ops.fetcher,
proxiedFetcher: ops.proxiedFetcher,
proxyStreams: ops.proxyStreams,
});
}

View File

@ -9,6 +9,10 @@ export const flags = {
// The source/embed is blocking cloudflare ip's
// This flag is not compatible with a proxy hosted on cloudflare
CF_BLOCKED: 'cf-blocked',
// Streams and sources with this flag wont be proxied
// And will be exclusive to the extension
PROXY_BLOCKED: 'proxy-blocked',
} as const;
export type Flags = (typeof flags)[keyof typeof flags];
@ -53,9 +57,14 @@ export const targetToFeatures: Record<Targets, FeatureMap> = {
},
};
export function getTargetFeatures(target: Targets, consistentIpForRequests: boolean): FeatureMap {
export function getTargetFeatures(
target: Targets,
consistentIpForRequests: boolean,
proxyStreams?: boolean,
): FeatureMap {
const features = targetToFeatures[target];
if (!consistentIpForRequests) features.disallowed.push(flags.IP_LOCKED);
if (proxyStreams) features.disallowed.push(flags.PROXY_BLOCKED);
return features;
}

View File

@ -1,6 +1,8 @@
import { load } from 'cheerio';
import { unpack } from 'unpacker';
import { flags } from '@/entrypoint/utils/targets';
import { SubtitleResult } from './types';
import { makeEmbed } from '../../base';
import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '../../captions';
@ -51,7 +53,7 @@ export const fileMoonScraper = makeEmbed({
id: 'primary',
type: 'hls',
playlist: file[1],
flags: [],
flags: [flags.IP_LOCKED],
captions,
},
],

View File

@ -1,3 +1,4 @@
import { flags } from '@/entrypoint/utils/targets';
import { NotFoundError } from '@/utils/errors';
import { makeEmbed } from '../../base';
@ -28,7 +29,7 @@ export const fileMoonMp4Scraper = makeEmbed({
url,
},
},
flags: [],
flags: [flags.IP_LOCKED],
captions: result.stream[0].captions,
},
],

View File

@ -1,9 +1,7 @@
import { load } from 'cheerio';
import crypto from 'crypto-js';
import { flags } from '@/entrypoint/utils/targets';
import { makeEmbed } from '@/providers/base';
import { convertPlaylistsToDataUrls } from '@/utils/playlist';
const { AES, MD5 } = crypto;
@ -115,9 +113,9 @@ export const playm4uNMScraper = makeEmbed({
{
id: 'primary',
type: 'hls',
playlist: await convertPlaylistsToDataUrls(ctx.proxiedFetcher, apiRes.data),
playlist: apiRes.data,
captions: [],
flags: [flags.CORS_ALLOWED],
flags: [],
},
],
};

View File

@ -1,3 +1,4 @@
import { flags } from '@/entrypoint/utils/targets';
import { makeEmbed } from '@/providers/base';
import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions';
@ -53,7 +54,7 @@ export const vidplayScraper = makeEmbed({
id: 'primary',
type: 'hls',
playlist: source,
flags: [],
flags: [flags.PROXY_BLOCKED],
headers: {
Referer: url.origin,
Origin: url.origin,

View File

@ -39,6 +39,7 @@ export const autoembedScraper = makeSourcerer({
id: 'autoembed',
name: 'Autoembed',
rank: 10,
disabled: true,
flags: [flags.CORS_ALLOWED],
scrapeMovie: comboScraper,
scrapeShow: comboScraper,

View File

@ -2,7 +2,6 @@
// thanks Paradox_77
import { load } from 'cheerio';
import { flags } from '@/entrypoint/utils/targets';
import { SourcererEmbed, makeSourcerer } from '@/providers/base';
import { compareMedia } from '@/utils/compare';
import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
@ -151,7 +150,7 @@ export const m4uScraper = makeSourcerer({
id: 'm4ufree',
name: 'M4UFree',
rank: 125,
flags: [flags.CORS_ALLOWED],
flags: [],
scrapeMovie: universalScraper,
scrapeShow: universalScraper,
});

View File

@ -31,6 +31,7 @@ export const smashyStreamScraper = makeSourcerer({
id: 'smashystream',
name: 'SmashyStream',
rank: 30,
disabled: true,
flags: [flags.CORS_ALLOWED],
scrapeMovie: universalScraper,
scrapeShow: universalScraper,

View File

@ -1,10 +1,9 @@
import { load } from 'cheerio';
import { flags } from '@/entrypoint/utils/targets';
import { Caption, labelToLanguageCode } from '@/providers/captions';
import { Stream } from '@/providers/streams';
import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
import { NotFoundError } from '@/utils/errors';
import { convertPlaylistsToDataUrls } from '@/utils/playlist';
import { InfoResponse } from './types';
import { SourcererOutput, makeSourcerer } from '../../base';
@ -89,20 +88,22 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext): Pr
stream: [
{
id: 'primary',
playlist: await convertPlaylistsToDataUrls(ctx.proxiedFetcher, `${baseUrl}/${streamResJson.val}`),
playlist: `${baseUrl}/${streamResJson.val}`,
type: 'hls',
flags: [flags.CORS_ALLOWED],
proxyDepth: 2,
flags: [],
captions,
},
...(streamResJson.val_bak
? [
{
id: 'backup',
playlist: await convertPlaylistsToDataUrls(ctx.proxiedFetcher, `${baseUrl}/${streamResJson.val_bak}`),
type: 'hls' as const,
flags: [flags.CORS_ALLOWED],
playlist: `${baseUrl}/${streamResJson.val_bak}`,
type: 'hls',
proxyDepth: 2,
flags: [],
captions,
},
} as Stream,
]
: []),
],
@ -113,7 +114,7 @@ export const soaperTvScraper = makeSourcerer({
id: 'soapertv',
name: 'SoaperTV',
rank: 126,
flags: [flags.CORS_ALLOWED],
flags: [],
scrapeMovie: universalScraper,
scrapeShow: universalScraper,
});

View File

@ -1,5 +1,6 @@
import { load } from 'cheerio';
import { flags } from '@/entrypoint/utils/targets';
import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base';
import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
@ -83,6 +84,6 @@ export const vidSrcToScraper = makeSourcerer({
name: 'VidSrcTo',
scrapeMovie: universalScraper,
scrapeShow: universalScraper,
flags: [],
flags: [flags.PROXY_BLOCKED],
rank: 130,
});

View File

@ -30,6 +30,7 @@ export type FileBasedStream = StreamCommon & {
export type HlsBasedStream = StreamCommon & {
type: 'hls';
playlist: string;
proxyDepth?: 0 | 1 | 2;
};
export type Stream = FileBasedStream | HlsBasedStream;

View File

@ -7,6 +7,7 @@ import { ProviderList } from '@/providers/get';
import { ScrapeContext } from '@/utils/context';
import { NotFoundError } from '@/utils/errors';
import { addOpenSubtitlesCaptions } from '@/utils/opensubtitles';
import { requiresProxy, setupProxy } from '@/utils/proxy';
import { isValidStream, validatePlayableStreams } from '@/utils/valid';
export type IndividualSourceRunnerOptions = {
@ -16,6 +17,7 @@ export type IndividualSourceRunnerOptions = {
media: ScrapeMedia;
id: string;
events?: IndividualScraperEvents;
proxyStreams?: boolean; // temporary
};
export async function scrapeInvidualSource(
@ -56,6 +58,10 @@ export async function scrapeInvidualSource(
output.stream = output.stream
.filter((stream) => isValidStream(stream))
.filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
output.stream = output.stream.map((stream) =>
requiresProxy(stream) && ops.proxyStreams ? setupProxy(stream) : stream,
);
}
if (!output) throw new Error('output is null');
@ -107,6 +113,7 @@ export type IndividualEmbedRunnerOptions = {
url: string;
id: string;
events?: IndividualScraperEvents;
proxyStreams?: boolean; // temporary
};
export async function scrapeIndividualEmbed(
@ -138,6 +145,10 @@ export async function scrapeIndividualEmbed(
.filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
if (output.stream.length === 0) throw new NotFoundError('No streams found');
output.stream = output.stream.map((stream) =>
requiresProxy(stream) && ops.proxyStreams ? setupProxy(stream) : stream,
);
const playableStreams = await validatePlayableStreams(output.stream, ops, embedScraper.id);
if (playableStreams.length === 0) throw new NotFoundError('No playable streams found');

View File

@ -9,6 +9,7 @@ import { ScrapeContext } from '@/utils/context';
import { NotFoundError } from '@/utils/errors';
import { reorderOnIdList } from '@/utils/list';
import { addOpenSubtitlesCaptions } from '@/utils/opensubtitles';
import { requiresProxy, setupProxy } from '@/utils/proxy';
import { isValidStream, validatePlayableStream } from '@/utils/valid';
export type RunOutput = {
@ -36,6 +37,7 @@ export type ProviderRunnerOptions = {
embedOrder?: string[];
events?: FullScraperEvents;
media: ScrapeMedia;
proxyStreams?: boolean; // temporary
};
export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOptions): Promise<RunOutput | null> {
@ -85,6 +87,10 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
output.stream = (output.stream ?? [])
.filter(isValidStream)
.filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
output.stream = output.stream.map((stream) =>
requiresProxy(stream) && ops.proxyStreams ? setupProxy(stream) : stream,
);
}
if (!output || (!output.stream?.length && !output.embeds.length)) {
throw new NotFoundError('No streams found');
@ -161,6 +167,9 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
embedOutput.stream = embedOutput.stream
.filter(isValidStream)
.filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
embedOutput.stream = embedOutput.stream.map((stream) =>
requiresProxy(stream) && ops.proxyStreams ? setupProxy(stream) : stream,
);
if (embedOutput.stream.length === 0) {
throw new NotFoundError('No streams found');
}

31
src/utils/proxy.ts Normal file
View File

@ -0,0 +1,31 @@
import { flags } from '@/entrypoint/utils/targets';
import { Stream } from '@/providers/streams';
export function requiresProxy(stream: Stream): boolean {
if (!stream.flags.includes(flags.CORS_ALLOWED) && !!(stream.headers && Object.keys(stream.headers).length > 0))
return true;
return false;
}
export function setupProxy(stream: Stream): Stream {
const headers =
stream.headers && Object.keys(stream.headers).length > 0 ? btoa(JSON.stringify(stream.headers)) : null;
const options = btoa(
JSON.stringify({
...(stream.type === 'hls' && { depth: stream.proxyDepth ?? 0 }),
}),
);
if (stream.type === 'hls')
stream.playlist = `https://proxy.nsbx.ru/hls/${btoa(stream.playlist)}/${headers}/${options}`;
if (stream.type === 'file')
Object.entries(stream.qualities).forEach((entry) => {
entry[1].url = `https://proxy.nsbx.ru/mp4/${btoa(entry[1].url)}/${headers}/${options}`;
});
stream.headers = {};
stream.flags = [flags.CORS_ALLOWED];
return stream;
}