Merge pull request #12 from ztpn/proxy

Merging this for now, won't enable it on the frontend till the proxy is ready

src/entrypoint/controls.ts

@@ -14,6 +14,7 @@ export interface ProviderControlsInput {
   features: FeatureMap;
   sources: Sourcerer[];
   embeds: Embed[];
+  proxyStreams?: boolean; // temporary
 }
 
 export interface RunnerOptions {
@@ -85,6 +86,7 @@ export function makeControls(ops: ProviderControlsInput): ProviderControls {
     features: ops.features,
     fetcher: makeFetcher(ops.fetcher),
     proxiedFetcher: makeFetcher(ops.proxiedFetcher ?? ops.fetcher),
+    proxyStreams: ops.proxyStreams,
   };
 
   return {

src/entrypoint/declare.ts

@@ -18,10 +18,17 @@ export interface ProviderMakerOptions {
   // Set this to true, if the requests will have the same IP as
   // the device that the stream will be played on
   consistentIpForRequests?: boolean;
+
+  // This is temporary
+  proxyStreams?: boolean;
 }
 
 export function makeProviders(ops: ProviderMakerOptions) {
-  const features = getTargetFeatures(ops.target, ops.consistentIpForRequests ?? false);
+  const features = getTargetFeatures(
+    ops.proxyStreams ? 'any' : ops.target,
+    ops.consistentIpForRequests ?? false,
+    ops.proxyStreams,
+  );
   const list = getProviders(features, {
     embeds: getBuiltinEmbeds(),
     sources: getBuiltinSources(),
@@ -33,5 +40,6 @@ export function makeProviders(ops: ProviderMakerOptions) {
     features,
     fetcher: ops.fetcher,
     proxiedFetcher: ops.proxiedFetcher,
+    proxyStreams: ops.proxyStreams,
   });
 }

src/entrypoint/utils/targets.ts

@@ -9,6 +9,10 @@ export const flags = {
   // The source/embed is blocking cloudflare ip's
   // This flag is not compatible with a proxy hosted on cloudflare
   CF_BLOCKED: 'cf-blocked',
+
+  // Streams and sources with this flag wont be proxied
+  // And will be exclusive to the extension
+  PROXY_BLOCKED: 'proxy-blocked',
 } as const;
 
 export type Flags = (typeof flags)[keyof typeof flags];
@@ -53,9 +57,14 @@ export const targetToFeatures: Record<Targets, FeatureMap> = {
   },
 };
 
-export function getTargetFeatures(target: Targets, consistentIpForRequests: boolean): FeatureMap {
+export function getTargetFeatures(
+  target: Targets,
+  consistentIpForRequests: boolean,
+  proxyStreams?: boolean,
+): FeatureMap {
   const features = targetToFeatures[target];
   if (!consistentIpForRequests) features.disallowed.push(flags.IP_LOCKED);
+  if (proxyStreams) features.disallowed.push(flags.PROXY_BLOCKED);
   return features;
 }
 

src/providers/embeds/filemoon/index.ts

@@ -1,6 +1,8 @@
 import { load } from 'cheerio';
 import { unpack } from 'unpacker';
 
+import { flags } from '@/entrypoint/utils/targets';
+
 import { SubtitleResult } from './types';
 import { makeEmbed } from '../../base';
 import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '../../captions';
@@ -51,7 +53,7 @@ export const fileMoonScraper = makeEmbed({
           id: 'primary',
           type: 'hls',
           playlist: file[1],
-          flags: [],
+          flags: [flags.IP_LOCKED],
           captions,
         },
       ],

src/providers/embeds/filemoon/mp4.ts

@@ -1,3 +1,4 @@
+import { flags } from '@/entrypoint/utils/targets';
 import { NotFoundError } from '@/utils/errors';
 
 import { makeEmbed } from '../../base';
@@ -28,7 +29,7 @@ export const fileMoonMp4Scraper = makeEmbed({
               url,
             },
           },
-          flags: [],
+          flags: [flags.IP_LOCKED],
           captions: result.stream[0].captions,
         },
       ],

src/providers/embeds/playm4u/nm.ts

@@ -1,9 +1,7 @@
 import { load } from 'cheerio';
 import crypto from 'crypto-js';
 
-import { flags } from '@/entrypoint/utils/targets';
 import { makeEmbed } from '@/providers/base';
-import { convertPlaylistsToDataUrls } from '@/utils/playlist';
 
 const { AES, MD5 } = crypto;
 
@@ -115,9 +113,9 @@ export const playm4uNMScraper = makeEmbed({
         {
           id: 'primary',
           type: 'hls',
-          playlist: await convertPlaylistsToDataUrls(ctx.proxiedFetcher, apiRes.data),
+          playlist: apiRes.data,
           captions: [],
-          flags: [flags.CORS_ALLOWED],
+          flags: [],
         },
       ],
     };

src/providers/embeds/vidplay/index.ts

@@ -1,3 +1,4 @@
+import { flags } from '@/entrypoint/utils/targets';
 import { makeEmbed } from '@/providers/base';
 import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions';
 
@@ -53,7 +54,7 @@ export const vidplayScraper = makeEmbed({
           id: 'primary',
           type: 'hls',
           playlist: source,
-          flags: [],
+          flags: [flags.PROXY_BLOCKED],
           headers: {
             Referer: url.origin,
             Origin: url.origin,

src/providers/sources/autoembed.ts

@@ -39,6 +39,7 @@ export const autoembedScraper = makeSourcerer({
   id: 'autoembed',
   name: 'Autoembed',
   rank: 10,
+  disabled: true,
   flags: [flags.CORS_ALLOWED],
   scrapeMovie: comboScraper,
   scrapeShow: comboScraper,

src/providers/sources/m4ufree.ts

@@ -2,7 +2,6 @@
 // thanks Paradox_77
 import { load } from 'cheerio';
 
-import { flags } from '@/entrypoint/utils/targets';
 import { SourcererEmbed, makeSourcerer } from '@/providers/base';
 import { compareMedia } from '@/utils/compare';
 import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
@@ -151,7 +150,7 @@ export const m4uScraper = makeSourcerer({
   id: 'm4ufree',
   name: 'M4UFree',
   rank: 125,
-  flags: [flags.CORS_ALLOWED],
+  flags: [],
   scrapeMovie: universalScraper,
   scrapeShow: universalScraper,
 });

src/providers/sources/smashystream/index.ts

@@ -31,6 +31,7 @@ export const smashyStreamScraper = makeSourcerer({
   id: 'smashystream',
   name: 'SmashyStream',
   rank: 30,
+  disabled: true,
   flags: [flags.CORS_ALLOWED],
   scrapeMovie: universalScraper,
   scrapeShow: universalScraper,

src/providers/sources/soapertv/index.ts

@@ -1,10 +1,9 @@
 import { load } from 'cheerio';
 
-import { flags } from '@/entrypoint/utils/targets';
 import { Caption, labelToLanguageCode } from '@/providers/captions';
+import { Stream } from '@/providers/streams';
 import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
 import { NotFoundError } from '@/utils/errors';
-import { convertPlaylistsToDataUrls } from '@/utils/playlist';
 
 import { InfoResponse } from './types';
 import { SourcererOutput, makeSourcerer } from '../../base';
@@ -89,20 +88,22 @@ const universalScraper = async (ctx: MovieScrapeContext | ShowScrapeContext): Pr
     stream: [
       {
         id: 'primary',
-        playlist: await convertPlaylistsToDataUrls(ctx.proxiedFetcher, `${baseUrl}/${streamResJson.val}`),
+        playlist: `${baseUrl}/${streamResJson.val}`,
         type: 'hls',
-        flags: [flags.CORS_ALLOWED],
+        proxyDepth: 2,
+        flags: [],
         captions,
       },
       ...(streamResJson.val_bak
         ? [
             {
               id: 'backup',
-              playlist: await convertPlaylistsToDataUrls(ctx.proxiedFetcher, `${baseUrl}/${streamResJson.val_bak}`),
+              playlist: `${baseUrl}/${streamResJson.val_bak}`,
-              type: 'hls' as const,
+              type: 'hls',
-              flags: [flags.CORS_ALLOWED],
+              proxyDepth: 2,
+              flags: [],
               captions,
-            },
+            } as Stream,
           ]
         : []),
     ],
@@ -113,7 +114,7 @@ export const soaperTvScraper = makeSourcerer({
   id: 'soapertv',
   name: 'SoaperTV',
   rank: 126,
-  flags: [flags.CORS_ALLOWED],
+  flags: [],
   scrapeMovie: universalScraper,
   scrapeShow: universalScraper,
 });

src/providers/sources/vidsrcto/index.ts

@@ -1,5 +1,6 @@
 import { load } from 'cheerio';
 
+import { flags } from '@/entrypoint/utils/targets';
 import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base';
 import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
 
@@ -83,6 +84,6 @@ export const vidSrcToScraper = makeSourcerer({
   name: 'VidSrcTo',
   scrapeMovie: universalScraper,
   scrapeShow: universalScraper,
-  flags: [],
+  flags: [flags.PROXY_BLOCKED],
   rank: 130,
 });

src/providers/streams.ts

@@ -30,6 +30,7 @@ export type FileBasedStream = StreamCommon & {
 export type HlsBasedStream = StreamCommon & {
   type: 'hls';
   playlist: string;
+  proxyDepth?: 0 | 1 | 2;
 };
 
 export type Stream = FileBasedStream | HlsBasedStream;

src/runners/individualRunner.ts

@@ -7,6 +7,7 @@ import { ProviderList } from '@/providers/get';
 import { ScrapeContext } from '@/utils/context';
 import { NotFoundError } from '@/utils/errors';
 import { addOpenSubtitlesCaptions } from '@/utils/opensubtitles';
+import { requiresProxy, setupProxy } from '@/utils/proxy';
 import { isValidStream, validatePlayableStreams } from '@/utils/valid';
 
 export type IndividualSourceRunnerOptions = {
@@ -16,6 +17,7 @@ export type IndividualSourceRunnerOptions = {
   media: ScrapeMedia;
   id: string;
   events?: IndividualScraperEvents;
+  proxyStreams?: boolean; // temporary
 };
 
 export async function scrapeInvidualSource(
@@ -56,6 +58,10 @@ export async function scrapeInvidualSource(
     output.stream = output.stream
       .filter((stream) => isValidStream(stream))
       .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
+
+    output.stream = output.stream.map((stream) =>
+      requiresProxy(stream) && ops.proxyStreams ? setupProxy(stream) : stream,
+    );
   }
 
   if (!output) throw new Error('output is null');
@@ -107,6 +113,7 @@ export type IndividualEmbedRunnerOptions = {
   url: string;
   id: string;
   events?: IndividualScraperEvents;
+  proxyStreams?: boolean; // temporary
 };
 
 export async function scrapeIndividualEmbed(
@@ -138,6 +145,10 @@ export async function scrapeIndividualEmbed(
     .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
   if (output.stream.length === 0) throw new NotFoundError('No streams found');
 
+  output.stream = output.stream.map((stream) =>
+    requiresProxy(stream) && ops.proxyStreams ? setupProxy(stream) : stream,
+  );
+
   const playableStreams = await validatePlayableStreams(output.stream, ops, embedScraper.id);
   if (playableStreams.length === 0) throw new NotFoundError('No playable streams found');
 

src/runners/runner.ts

@@ -9,6 +9,7 @@ import { ScrapeContext } from '@/utils/context';
 import { NotFoundError } from '@/utils/errors';
 import { reorderOnIdList } from '@/utils/list';
 import { addOpenSubtitlesCaptions } from '@/utils/opensubtitles';
+import { requiresProxy, setupProxy } from '@/utils/proxy';
 import { isValidStream, validatePlayableStream } from '@/utils/valid';
 
 export type RunOutput = {
@@ -36,6 +37,7 @@ export type ProviderRunnerOptions = {
   embedOrder?: string[];
   events?: FullScraperEvents;
   media: ScrapeMedia;
+  proxyStreams?: boolean; // temporary
 };
 
 export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOptions): Promise<RunOutput | null> {
@@ -85,6 +87,10 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
         output.stream = (output.stream ?? [])
           .filter(isValidStream)
           .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
+
+        output.stream = output.stream.map((stream) =>
+          requiresProxy(stream) && ops.proxyStreams ? setupProxy(stream) : stream,
+        );
       }
       if (!output || (!output.stream?.length && !output.embeds.length)) {
         throw new NotFoundError('No streams found');
@@ -161,6 +167,9 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
         embedOutput.stream = embedOutput.stream
           .filter(isValidStream)
           .filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
+        embedOutput.stream = embedOutput.stream.map((stream) =>
+          requiresProxy(stream) && ops.proxyStreams ? setupProxy(stream) : stream,
+        );
         if (embedOutput.stream.length === 0) {
           throw new NotFoundError('No streams found');
         }

src/utils/proxy.ts

@@ -0,0 +1,31 @@
+import { flags } from '@/entrypoint/utils/targets';
+import { Stream } from '@/providers/streams';
+
+export function requiresProxy(stream: Stream): boolean {
+  if (!stream.flags.includes(flags.CORS_ALLOWED) && !!(stream.headers && Object.keys(stream.headers).length > 0))
+    return true;
+  return false;
+}
+
+export function setupProxy(stream: Stream): Stream {
+  const headers =
+    stream.headers && Object.keys(stream.headers).length > 0 ? btoa(JSON.stringify(stream.headers)) : null;
+
+  const options = btoa(
+    JSON.stringify({
+      ...(stream.type === 'hls' && { depth: stream.proxyDepth ?? 0 }),
+    }),
+  );
+
+  if (stream.type === 'hls')
+    stream.playlist = `https://proxy.nsbx.ru/hls/${btoa(stream.playlist)}/${headers}/${options}`;
+
+  if (stream.type === 'file')
+    Object.entries(stream.qualities).forEach((entry) => {
+      entry[1].url = `https://proxy.nsbx.ru/mp4/${btoa(entry[1].url)}/${headers}/${options}`;
+    });
+
+  stream.headers = {};
+  stream.flags = [flags.CORS_ALLOWED];
+  return stream;
+}