merge with dev

README.md

@@ -26,7 +26,6 @@ The following CLI Mode arguments are available
 | `--season`    | `-s`   | Season number. Only used if type is `show`                              | `0`          |
 | `--episode`   | `-e`   | Episode number. Only used if type is `show`                             | `0`          |
 | `--url`       | `-u`   | URL to a video embed. Only used if source is an embed                   |              |
-| `--headers`   | `-h`   | Optional headers to send while scraping                                 |              |
 | `--help`      | `-h`   | Shows help for the command arguments                                    |              |
 
 Example testing the FlixHQ source on the movie "Spirited Away"

src/fetchers/common.ts

@@ -35,7 +35,6 @@ export function makeFetcher(fetcher: Fetcher): UseableFetcher {
       baseUrl: ops?.baseUrl ?? '',
       readHeaders: ops?.readHeaders ?? [],
       body: ops?.body,
-      returnRaw: ops?.returnRaw ?? false,
     });
   };
   const output: UseableFetcher = async (url, ops) => (await newFetcher(url, ops)).body;

src/fetchers/standardFetch.ts

@@ -29,10 +29,6 @@ export function makeStandardFetcher(f: FetchLike): Fetcher {
       body: seralizedBody.body,
     });
 
-    if (ops.returnRaw) {
-      return res;
-    }
-
     let body: any;
     const isJson = res.headers.get('content-type')?.includes('application/json');
     if (isJson) body = await res.json();

src/fetchers/types.ts

@@ -7,7 +7,6 @@ export type FetcherOptions = {
   method?: 'HEAD' | 'GET' | 'POST';
   readHeaders?: string[];
   body?: Record<string, any> | string | FormData | URLSearchParams;
-  returnRaw?: boolean;
 };
 
 // Version of the options that always has the defaults set
@@ -18,7 +17,6 @@ export type DefaultedFetcherOptions = {
   headers: Record<string, string>;
   query: Record<string, string>;
   method: 'HEAD' | 'GET' | 'POST';
-  returnRaw: boolean;
   readHeaders: string[];
 };
 

src/providers/embeds/streambucket.ts

@@ -1,4 +1,4 @@
-import { flags } from '@/main/targets';
+import { flags } from '@/index';
 import { makeEmbed } from '@/providers/base';
 
 // StreamBucket makes use of https://github.com/nicxlau/hunter-php-javascript-obfuscator
@@ -87,12 +87,15 @@ export const streambucketScraper = makeEmbed({
     }
 
     return {
-      stream: {
+      stream: [
+        {
+          id: 'primary',
           type: 'hls',
           playlist: regexResult[1],
-        flags: [flags.NO_CORS],
+          flags: [flags.CORS_ALLOWED],
           captions: [],
         },
+      ],
     };
   },
 });

src/providers/embeds/vidsrc.ts

@@ -1,3 +1,4 @@
+import { flags } from '@/../lib';
 import { makeEmbed } from '@/providers/base';
 
 const hlsURLRegex = /file:"(.*?)"/;
@@ -7,11 +8,10 @@ export const vidsrcembedScraper = makeEmbed({
   name: 'VidSrc',
   rank: 197,
   async scrape(ctx) {
-    if (!ctx.headers || (!ctx.headers.referer && !ctx.headers.Referer)) {
-      throw new Error('VidSrc embeds require the referer header to be set');
-    }
     const html = await ctx.proxiedFetcher<string>(ctx.url, {
-      headers: ctx.headers,
+      headers: {
+        referer: ctx.url,
+      },
     });
 
     const match = html
@@ -24,12 +24,15 @@ export const vidsrcembedScraper = makeEmbed({
     if (!finalUrl.includes('.m3u8')) throw new Error('Unable to find HLS playlist');
 
     return {
-      stream: {
+      stream: [
+        {
+          id: 'primary',
           type: 'hls',
           playlist: finalUrl,
-        flags: [],
+          flags: [flags.CORS_ALLOWED],
           captions: [],
         },
+      ],
     };
   },
 });

src/providers/sources/vidsrc/index.ts

@@ -1,4 +1,4 @@
-import { flags } from '@/main/targets';
+import { flags } from '@/entrypoint/utils/targets';
 import { makeSourcerer } from '@/providers/base';
 import { scrapeMovie } from '@/providers/sources/vidsrc/scrape-movie';
 import { scrapeShow } from '@/providers/sources/vidsrc/scrape-show';
@@ -7,7 +7,7 @@ export const vidsrcScraper = makeSourcerer({
   id: 'vidsrc',
   name: 'VidSrc',
   rank: 120,
-  flags: [flags.NO_CORS],
+  flags: [flags.CORS_ALLOWED],
   scrapeMovie,
   scrapeShow,
 });

src/providers/sources/vidsrc/scrape.ts

@@ -1,6 +1,5 @@
 import { load } from 'cheerio';
 
-import { FetchReply } from '@/fetchers/fetch';
 import { SourcererEmbed } from '@/providers/base';
 import { streambucketScraper } from '@/providers/embeds/streambucket';
 import { vidsrcembedScraper } from '@/providers/embeds/vidsrc';
@@ -44,7 +43,7 @@ async function getVidSrcEmbeds(ctx: MovieScrapeContext | ShowScrapeContext, star
     html = await ctx.proxiedFetcher<string>(`/rcp/${hash}`, {
       baseUrl: vidsrcRCPBase,
       headers: {
-        referer: `${vidsrcBase}${startingURL}`,
+        referer: vidsrcBase,
       },
     });
 
@@ -61,32 +60,23 @@ async function getVidSrcEmbeds(ctx: MovieScrapeContext | ShowScrapeContext, star
       redirectURL = `https:${redirectURL}`;
     }
 
-    // Return the raw fetch response here.
+    const { finalUrl } = await ctx.proxiedFetcher.full(redirectURL, {
-    // When a Location header is sent, fetch
+      method: 'HEAD',
-    // will silently follow it. The "url" inside
-    // the Response is the final requested URL,
-    // which is the real embeds URL
-    const { url: embedURL } = await ctx.proxiedFetcher<FetchReply>(redirectURL, {
-      returnRaw: true,
-      method: 'HEAD', // We don't care about the actual response body here
       headers: {
-        referer: `${vidsrcRCPBase}/rcp/${hash}`,
+        referer: vidsrcBase,
       },
     });
 
     const embed: SourcererEmbed = {
       embedId: '',
-      url: embedURL,
+      url: finalUrl,
     };
 
-    const parsedUrl = new URL(embedURL);
+    const parsedUrl = new URL(finalUrl);
 
     switch (parsedUrl.host) {
       case 'vidsrc.stream':
         embed.embedId = vidsrcembedScraper.id;
-        embed.headers = {
-          referer: `${vidsrcRCPBase}/rcp/${hash}`,
-        };
         break;
       case 'streambucket.net':
         embed.embedId = streambucketScraper.id;
@@ -99,7 +89,7 @@ async function getVidSrcEmbeds(ctx: MovieScrapeContext | ShowScrapeContext, star
         // Just ignore this. This embed streams video over a custom WebSocket connection
         break;
       default:
-        throw new Error(`Failed to find VidSrc embed source for ${embedURL}`);
+        throw new Error(`Failed to find VidSrc embed source for ${finalUrl}`);
     }
 
     // Since some embeds are ignored on purpose, check if a valid one was found

src/utils/context.ts

@@ -2,8 +2,14 @@ import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media';
 import { UseableFetcher } from '@/fetchers/types';
 
 export type ScrapeContext = {
-  proxiedFetcher: <T>(...params: Parameters<UseableFetcher<T>>) => ReturnType<UseableFetcher<T>>;
+  proxiedFetcher: {
-  fetcher: <T>(...params: Parameters<UseableFetcher<T>>) => ReturnType<UseableFetcher<T>>;
+    <T>(...params: Parameters<UseableFetcher<T>>): ReturnType<UseableFetcher<T>>;
+    full<T>(...params: Parameters<UseableFetcher<T>['full']>): ReturnType<UseableFetcher<T>['full']>;
+  };
+  fetcher: {
+    <T>(...params: Parameters<UseableFetcher<T>>): ReturnType<UseableFetcher<T>>;
+    full<T>(...params: Parameters<UseableFetcher<T>['full']>): ReturnType<UseableFetcher<T>['full']>;
+  };
   progress(val: number): void;
 };