merge with dev
This commit is contained in:
parent
304ef68c5f
commit
65bbf28442
|
@ -26,7 +26,6 @@ The following CLI Mode arguments are available
|
|||
| `--season` | `-s` | Season number. Only used if type is `show` | `0` |
|
||||
| `--episode` | `-e` | Episode number. Only used if type is `show` | `0` |
|
||||
| `--url` | `-u` | URL to a video embed. Only used if source is an embed | |
|
||||
| `--headers` | `-h` | Optional headers to send while scraping | |
|
||||
| `--help` | `-h` | Shows help for the command arguments | |
|
||||
|
||||
Example testing the FlixHQ source on the movie "Spirited Away"
|
||||
|
|
|
@ -35,7 +35,6 @@ export function makeFetcher(fetcher: Fetcher): UseableFetcher {
|
|||
baseUrl: ops?.baseUrl ?? '',
|
||||
readHeaders: ops?.readHeaders ?? [],
|
||||
body: ops?.body,
|
||||
returnRaw: ops?.returnRaw ?? false,
|
||||
});
|
||||
};
|
||||
const output: UseableFetcher = async (url, ops) => (await newFetcher(url, ops)).body;
|
||||
|
|
|
@ -29,10 +29,6 @@ export function makeStandardFetcher(f: FetchLike): Fetcher {
|
|||
body: seralizedBody.body,
|
||||
});
|
||||
|
||||
if (ops.returnRaw) {
|
||||
return res;
|
||||
}
|
||||
|
||||
let body: any;
|
||||
const isJson = res.headers.get('content-type')?.includes('application/json');
|
||||
if (isJson) body = await res.json();
|
||||
|
|
|
@ -7,7 +7,6 @@ export type FetcherOptions = {
|
|||
method?: 'HEAD' | 'GET' | 'POST';
|
||||
readHeaders?: string[];
|
||||
body?: Record<string, any> | string | FormData | URLSearchParams;
|
||||
returnRaw?: boolean;
|
||||
};
|
||||
|
||||
// Version of the options that always has the defaults set
|
||||
|
@ -18,7 +17,6 @@ export type DefaultedFetcherOptions = {
|
|||
headers: Record<string, string>;
|
||||
query: Record<string, string>;
|
||||
method: 'HEAD' | 'GET' | 'POST';
|
||||
returnRaw: boolean;
|
||||
readHeaders: string[];
|
||||
};
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import { flags } from '@/main/targets';
|
||||
import { flags } from '@/index';
|
||||
import { makeEmbed } from '@/providers/base';
|
||||
|
||||
// StreamBucket makes use of https://github.com/nicxlau/hunter-php-javascript-obfuscator
|
||||
|
@ -87,12 +87,15 @@ export const streambucketScraper = makeEmbed({
|
|||
}
|
||||
|
||||
return {
|
||||
stream: {
|
||||
type: 'hls',
|
||||
playlist: regexResult[1],
|
||||
flags: [flags.NO_CORS],
|
||||
captions: [],
|
||||
},
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: regexResult[1],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
captions: [],
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
});
|
||||
|
|
|
@ -1,3 +1,4 @@
|
|||
import { flags } from '@/../lib';
|
||||
import { makeEmbed } from '@/providers/base';
|
||||
|
||||
const hlsURLRegex = /file:"(.*?)"/;
|
||||
|
@ -7,11 +8,10 @@ export const vidsrcembedScraper = makeEmbed({
|
|||
name: 'VidSrc',
|
||||
rank: 197,
|
||||
async scrape(ctx) {
|
||||
if (!ctx.headers || (!ctx.headers.referer && !ctx.headers.Referer)) {
|
||||
throw new Error('VidSrc embeds require the referer header to be set');
|
||||
}
|
||||
const html = await ctx.proxiedFetcher<string>(ctx.url, {
|
||||
headers: ctx.headers,
|
||||
headers: {
|
||||
referer: ctx.url,
|
||||
},
|
||||
});
|
||||
|
||||
const match = html
|
||||
|
@ -24,12 +24,15 @@ export const vidsrcembedScraper = makeEmbed({
|
|||
if (!finalUrl.includes('.m3u8')) throw new Error('Unable to find HLS playlist');
|
||||
|
||||
return {
|
||||
stream: {
|
||||
type: 'hls',
|
||||
playlist: finalUrl,
|
||||
flags: [],
|
||||
captions: [],
|
||||
},
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: finalUrl,
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
captions: [],
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
});
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import { flags } from '@/main/targets';
|
||||
import { flags } from '@/entrypoint/utils/targets';
|
||||
import { makeSourcerer } from '@/providers/base';
|
||||
import { scrapeMovie } from '@/providers/sources/vidsrc/scrape-movie';
|
||||
import { scrapeShow } from '@/providers/sources/vidsrc/scrape-show';
|
||||
|
@ -7,7 +7,7 @@ export const vidsrcScraper = makeSourcerer({
|
|||
id: 'vidsrc',
|
||||
name: 'VidSrc',
|
||||
rank: 120,
|
||||
flags: [flags.NO_CORS],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
scrapeMovie,
|
||||
scrapeShow,
|
||||
});
|
||||
|
|
|
@ -1,6 +1,5 @@
|
|||
import { load } from 'cheerio';
|
||||
|
||||
import { FetchReply } from '@/fetchers/fetch';
|
||||
import { SourcererEmbed } from '@/providers/base';
|
||||
import { streambucketScraper } from '@/providers/embeds/streambucket';
|
||||
import { vidsrcembedScraper } from '@/providers/embeds/vidsrc';
|
||||
|
@ -44,7 +43,7 @@ async function getVidSrcEmbeds(ctx: MovieScrapeContext | ShowScrapeContext, star
|
|||
html = await ctx.proxiedFetcher<string>(`/rcp/${hash}`, {
|
||||
baseUrl: vidsrcRCPBase,
|
||||
headers: {
|
||||
referer: `${vidsrcBase}${startingURL}`,
|
||||
referer: vidsrcBase,
|
||||
},
|
||||
});
|
||||
|
||||
|
@ -61,32 +60,23 @@ async function getVidSrcEmbeds(ctx: MovieScrapeContext | ShowScrapeContext, star
|
|||
redirectURL = `https:${redirectURL}`;
|
||||
}
|
||||
|
||||
// Return the raw fetch response here.
|
||||
// When a Location header is sent, fetch
|
||||
// will silently follow it. The "url" inside
|
||||
// the Response is the final requested URL,
|
||||
// which is the real embeds URL
|
||||
const { url: embedURL } = await ctx.proxiedFetcher<FetchReply>(redirectURL, {
|
||||
returnRaw: true,
|
||||
method: 'HEAD', // We don't care about the actual response body here
|
||||
const { finalUrl } = await ctx.proxiedFetcher.full(redirectURL, {
|
||||
method: 'HEAD',
|
||||
headers: {
|
||||
referer: `${vidsrcRCPBase}/rcp/${hash}`,
|
||||
referer: vidsrcBase,
|
||||
},
|
||||
});
|
||||
|
||||
const embed: SourcererEmbed = {
|
||||
embedId: '',
|
||||
url: embedURL,
|
||||
url: finalUrl,
|
||||
};
|
||||
|
||||
const parsedUrl = new URL(embedURL);
|
||||
const parsedUrl = new URL(finalUrl);
|
||||
|
||||
switch (parsedUrl.host) {
|
||||
case 'vidsrc.stream':
|
||||
embed.embedId = vidsrcembedScraper.id;
|
||||
embed.headers = {
|
||||
referer: `${vidsrcRCPBase}/rcp/${hash}`,
|
||||
};
|
||||
break;
|
||||
case 'streambucket.net':
|
||||
embed.embedId = streambucketScraper.id;
|
||||
|
@ -99,7 +89,7 @@ async function getVidSrcEmbeds(ctx: MovieScrapeContext | ShowScrapeContext, star
|
|||
// Just ignore this. This embed streams video over a custom WebSocket connection
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Failed to find VidSrc embed source for ${embedURL}`);
|
||||
throw new Error(`Failed to find VidSrc embed source for ${finalUrl}`);
|
||||
}
|
||||
|
||||
// Since some embeds are ignored on purpose, check if a valid one was found
|
||||
|
|
|
@ -2,8 +2,14 @@ import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media';
|
|||
import { UseableFetcher } from '@/fetchers/types';
|
||||
|
||||
export type ScrapeContext = {
|
||||
proxiedFetcher: <T>(...params: Parameters<UseableFetcher<T>>) => ReturnType<UseableFetcher<T>>;
|
||||
fetcher: <T>(...params: Parameters<UseableFetcher<T>>) => ReturnType<UseableFetcher<T>>;
|
||||
proxiedFetcher: {
|
||||
<T>(...params: Parameters<UseableFetcher<T>>): ReturnType<UseableFetcher<T>>;
|
||||
full<T>(...params: Parameters<UseableFetcher<T>['full']>): ReturnType<UseableFetcher<T>['full']>;
|
||||
};
|
||||
fetcher: {
|
||||
<T>(...params: Parameters<UseableFetcher<T>>): ReturnType<UseableFetcher<T>>;
|
||||
full<T>(...params: Parameters<UseableFetcher<T>['full']>): ReturnType<UseableFetcher<T>['full']>;
|
||||
};
|
||||
progress(val: number): void;
|
||||
};
|
||||
|
||||
|
|
Loading…
Reference in New Issue