diff --git a/README.md b/README.md index 2b40e6e..8b7633c 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,6 @@ The following CLI Mode arguments are available | `--season` | `-s` | Season number. Only used if type is `show` | `0` | | `--episode` | `-e` | Episode number. Only used if type is `show` | `0` | | `--url` | `-u` | URL to a video embed. Only used if source is an embed | | -| `--headers` | `-h` | Optional headers to send while scraping | | | `--help` | `-h` | Shows help for the command arguments | | Example testing the FlixHQ source on the movie "Spirited Away" diff --git a/src/fetchers/common.ts b/src/fetchers/common.ts index 90e87ce..71956ba 100644 --- a/src/fetchers/common.ts +++ b/src/fetchers/common.ts @@ -35,7 +35,6 @@ export function makeFetcher(fetcher: Fetcher): UseableFetcher { baseUrl: ops?.baseUrl ?? '', readHeaders: ops?.readHeaders ?? [], body: ops?.body, - returnRaw: ops?.returnRaw ?? false, }); }; const output: UseableFetcher = async (url, ops) => (await newFetcher(url, ops)).body; diff --git a/src/fetchers/standardFetch.ts b/src/fetchers/standardFetch.ts index 4eba0d0..9fb6afa 100644 --- a/src/fetchers/standardFetch.ts +++ b/src/fetchers/standardFetch.ts @@ -29,10 +29,6 @@ export function makeStandardFetcher(f: FetchLike): Fetcher { body: seralizedBody.body, }); - if (ops.returnRaw) { - return res; - } - let body: any; const isJson = res.headers.get('content-type')?.includes('application/json'); if (isJson) body = await res.json(); diff --git a/src/fetchers/types.ts b/src/fetchers/types.ts index 09640aa..8904173 100644 --- a/src/fetchers/types.ts +++ b/src/fetchers/types.ts @@ -7,7 +7,6 @@ export type FetcherOptions = { method?: 'HEAD' | 'GET' | 'POST'; readHeaders?: string[]; body?: Record | string | FormData | URLSearchParams; - returnRaw?: boolean; }; // Version of the options that always has the defaults set @@ -18,7 +17,6 @@ export type DefaultedFetcherOptions = { headers: Record; query: Record; method: 'HEAD' | 'GET' | 'POST'; - returnRaw: boolean; readHeaders: string[]; }; diff --git a/src/providers/embeds/streambucket.ts b/src/providers/embeds/streambucket.ts index f137ceb..f22ad42 100644 --- a/src/providers/embeds/streambucket.ts +++ b/src/providers/embeds/streambucket.ts @@ -1,4 +1,4 @@ -import { flags } from '@/main/targets'; +import { flags } from '@/index'; import { makeEmbed } from '@/providers/base'; // StreamBucket makes use of https://github.com/nicxlau/hunter-php-javascript-obfuscator @@ -87,12 +87,15 @@ export const streambucketScraper = makeEmbed({ } return { - stream: { - type: 'hls', - playlist: regexResult[1], - flags: [flags.NO_CORS], - captions: [], - }, + stream: [ + { + id: 'primary', + type: 'hls', + playlist: regexResult[1], + flags: [flags.CORS_ALLOWED], + captions: [], + }, + ], }; }, }); diff --git a/src/providers/embeds/vidsrc.ts b/src/providers/embeds/vidsrc.ts index da3f2aa..efd3a18 100644 --- a/src/providers/embeds/vidsrc.ts +++ b/src/providers/embeds/vidsrc.ts @@ -1,3 +1,4 @@ +import { flags } from '@/../lib'; import { makeEmbed } from '@/providers/base'; const hlsURLRegex = /file:"(.*?)"/; @@ -7,11 +8,10 @@ export const vidsrcembedScraper = makeEmbed({ name: 'VidSrc', rank: 197, async scrape(ctx) { - if (!ctx.headers || (!ctx.headers.referer && !ctx.headers.Referer)) { - throw new Error('VidSrc embeds require the referer header to be set'); - } const html = await ctx.proxiedFetcher(ctx.url, { - headers: ctx.headers, + headers: { + referer: ctx.url, + }, }); const match = html @@ -24,12 +24,15 @@ export const vidsrcembedScraper = makeEmbed({ if (!finalUrl.includes('.m3u8')) throw new Error('Unable to find HLS playlist'); return { - stream: { - type: 'hls', - playlist: finalUrl, - flags: [], - captions: [], - }, + stream: [ + { + id: 'primary', + type: 'hls', + playlist: finalUrl, + flags: [flags.CORS_ALLOWED], + captions: [], + }, + ], }; }, }); diff --git a/src/providers/sources/vidsrc/index.ts b/src/providers/sources/vidsrc/index.ts index f7ad792..6331a05 100644 --- a/src/providers/sources/vidsrc/index.ts +++ b/src/providers/sources/vidsrc/index.ts @@ -1,4 +1,4 @@ -import { flags } from '@/main/targets'; +import { flags } from '@/entrypoint/utils/targets'; import { makeSourcerer } from '@/providers/base'; import { scrapeMovie } from '@/providers/sources/vidsrc/scrape-movie'; import { scrapeShow } from '@/providers/sources/vidsrc/scrape-show'; @@ -7,7 +7,7 @@ export const vidsrcScraper = makeSourcerer({ id: 'vidsrc', name: 'VidSrc', rank: 120, - flags: [flags.NO_CORS], + flags: [flags.CORS_ALLOWED], scrapeMovie, scrapeShow, }); diff --git a/src/providers/sources/vidsrc/scrape.ts b/src/providers/sources/vidsrc/scrape.ts index 6ea5256..65a709c 100644 --- a/src/providers/sources/vidsrc/scrape.ts +++ b/src/providers/sources/vidsrc/scrape.ts @@ -1,6 +1,5 @@ import { load } from 'cheerio'; -import { FetchReply } from '@/fetchers/fetch'; import { SourcererEmbed } from '@/providers/base'; import { streambucketScraper } from '@/providers/embeds/streambucket'; import { vidsrcembedScraper } from '@/providers/embeds/vidsrc'; @@ -44,7 +43,7 @@ async function getVidSrcEmbeds(ctx: MovieScrapeContext | ShowScrapeContext, star html = await ctx.proxiedFetcher(`/rcp/${hash}`, { baseUrl: vidsrcRCPBase, headers: { - referer: `${vidsrcBase}${startingURL}`, + referer: vidsrcBase, }, }); @@ -61,32 +60,23 @@ async function getVidSrcEmbeds(ctx: MovieScrapeContext | ShowScrapeContext, star redirectURL = `https:${redirectURL}`; } - // Return the raw fetch response here. - // When a Location header is sent, fetch - // will silently follow it. The "url" inside - // the Response is the final requested URL, - // which is the real embeds URL - const { url: embedURL } = await ctx.proxiedFetcher(redirectURL, { - returnRaw: true, - method: 'HEAD', // We don't care about the actual response body here + const { finalUrl } = await ctx.proxiedFetcher.full(redirectURL, { + method: 'HEAD', headers: { - referer: `${vidsrcRCPBase}/rcp/${hash}`, + referer: vidsrcBase, }, }); const embed: SourcererEmbed = { embedId: '', - url: embedURL, + url: finalUrl, }; - const parsedUrl = new URL(embedURL); + const parsedUrl = new URL(finalUrl); switch (parsedUrl.host) { case 'vidsrc.stream': embed.embedId = vidsrcembedScraper.id; - embed.headers = { - referer: `${vidsrcRCPBase}/rcp/${hash}`, - }; break; case 'streambucket.net': embed.embedId = streambucketScraper.id; @@ -99,7 +89,7 @@ async function getVidSrcEmbeds(ctx: MovieScrapeContext | ShowScrapeContext, star // Just ignore this. This embed streams video over a custom WebSocket connection break; default: - throw new Error(`Failed to find VidSrc embed source for ${embedURL}`); + throw new Error(`Failed to find VidSrc embed source for ${finalUrl}`); } // Since some embeds are ignored on purpose, check if a valid one was found diff --git a/src/utils/context.ts b/src/utils/context.ts index cd11f21..f7c005e 100644 --- a/src/utils/context.ts +++ b/src/utils/context.ts @@ -2,8 +2,14 @@ import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media'; import { UseableFetcher } from '@/fetchers/types'; export type ScrapeContext = { - proxiedFetcher: (...params: Parameters>) => ReturnType>; - fetcher: (...params: Parameters>) => ReturnType>; + proxiedFetcher: { + (...params: Parameters>): ReturnType>; + full(...params: Parameters['full']>): ReturnType['full']>; + }; + fetcher: { + (...params: Parameters>): ReturnType>; + full(...params: Parameters['full']>): ReturnType['full']>; + }; progress(val: number): void; };