merge with dev
This commit is contained in:
parent
304ef68c5f
commit
65bbf28442
|
@ -26,7 +26,6 @@ The following CLI Mode arguments are available
|
||||||
| `--season` | `-s` | Season number. Only used if type is `show` | `0` |
|
| `--season` | `-s` | Season number. Only used if type is `show` | `0` |
|
||||||
| `--episode` | `-e` | Episode number. Only used if type is `show` | `0` |
|
| `--episode` | `-e` | Episode number. Only used if type is `show` | `0` |
|
||||||
| `--url` | `-u` | URL to a video embed. Only used if source is an embed | |
|
| `--url` | `-u` | URL to a video embed. Only used if source is an embed | |
|
||||||
| `--headers` | `-h` | Optional headers to send while scraping | |
|
|
||||||
| `--help` | `-h` | Shows help for the command arguments | |
|
| `--help` | `-h` | Shows help for the command arguments | |
|
||||||
|
|
||||||
Example testing the FlixHQ source on the movie "Spirited Away"
|
Example testing the FlixHQ source on the movie "Spirited Away"
|
||||||
|
|
|
@ -35,7 +35,6 @@ export function makeFetcher(fetcher: Fetcher): UseableFetcher {
|
||||||
baseUrl: ops?.baseUrl ?? '',
|
baseUrl: ops?.baseUrl ?? '',
|
||||||
readHeaders: ops?.readHeaders ?? [],
|
readHeaders: ops?.readHeaders ?? [],
|
||||||
body: ops?.body,
|
body: ops?.body,
|
||||||
returnRaw: ops?.returnRaw ?? false,
|
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
const output: UseableFetcher = async (url, ops) => (await newFetcher(url, ops)).body;
|
const output: UseableFetcher = async (url, ops) => (await newFetcher(url, ops)).body;
|
||||||
|
|
|
@ -29,10 +29,6 @@ export function makeStandardFetcher(f: FetchLike): Fetcher {
|
||||||
body: seralizedBody.body,
|
body: seralizedBody.body,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (ops.returnRaw) {
|
|
||||||
return res;
|
|
||||||
}
|
|
||||||
|
|
||||||
let body: any;
|
let body: any;
|
||||||
const isJson = res.headers.get('content-type')?.includes('application/json');
|
const isJson = res.headers.get('content-type')?.includes('application/json');
|
||||||
if (isJson) body = await res.json();
|
if (isJson) body = await res.json();
|
||||||
|
|
|
@ -7,7 +7,6 @@ export type FetcherOptions = {
|
||||||
method?: 'HEAD' | 'GET' | 'POST';
|
method?: 'HEAD' | 'GET' | 'POST';
|
||||||
readHeaders?: string[];
|
readHeaders?: string[];
|
||||||
body?: Record<string, any> | string | FormData | URLSearchParams;
|
body?: Record<string, any> | string | FormData | URLSearchParams;
|
||||||
returnRaw?: boolean;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
// Version of the options that always has the defaults set
|
// Version of the options that always has the defaults set
|
||||||
|
@ -18,7 +17,6 @@ export type DefaultedFetcherOptions = {
|
||||||
headers: Record<string, string>;
|
headers: Record<string, string>;
|
||||||
query: Record<string, string>;
|
query: Record<string, string>;
|
||||||
method: 'HEAD' | 'GET' | 'POST';
|
method: 'HEAD' | 'GET' | 'POST';
|
||||||
returnRaw: boolean;
|
|
||||||
readHeaders: string[];
|
readHeaders: string[];
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
import { flags } from '@/main/targets';
|
import { flags } from '@/index';
|
||||||
import { makeEmbed } from '@/providers/base';
|
import { makeEmbed } from '@/providers/base';
|
||||||
|
|
||||||
// StreamBucket makes use of https://github.com/nicxlau/hunter-php-javascript-obfuscator
|
// StreamBucket makes use of https://github.com/nicxlau/hunter-php-javascript-obfuscator
|
||||||
|
@ -87,12 +87,15 @@ export const streambucketScraper = makeEmbed({
|
||||||
}
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
stream: {
|
stream: [
|
||||||
type: 'hls',
|
{
|
||||||
playlist: regexResult[1],
|
id: 'primary',
|
||||||
flags: [flags.NO_CORS],
|
type: 'hls',
|
||||||
captions: [],
|
playlist: regexResult[1],
|
||||||
},
|
flags: [flags.CORS_ALLOWED],
|
||||||
|
captions: [],
|
||||||
|
},
|
||||||
|
],
|
||||||
};
|
};
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
import { flags } from '@/../lib';
|
||||||
import { makeEmbed } from '@/providers/base';
|
import { makeEmbed } from '@/providers/base';
|
||||||
|
|
||||||
const hlsURLRegex = /file:"(.*?)"/;
|
const hlsURLRegex = /file:"(.*?)"/;
|
||||||
|
@ -7,11 +8,10 @@ export const vidsrcembedScraper = makeEmbed({
|
||||||
name: 'VidSrc',
|
name: 'VidSrc',
|
||||||
rank: 197,
|
rank: 197,
|
||||||
async scrape(ctx) {
|
async scrape(ctx) {
|
||||||
if (!ctx.headers || (!ctx.headers.referer && !ctx.headers.Referer)) {
|
|
||||||
throw new Error('VidSrc embeds require the referer header to be set');
|
|
||||||
}
|
|
||||||
const html = await ctx.proxiedFetcher<string>(ctx.url, {
|
const html = await ctx.proxiedFetcher<string>(ctx.url, {
|
||||||
headers: ctx.headers,
|
headers: {
|
||||||
|
referer: ctx.url,
|
||||||
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
const match = html
|
const match = html
|
||||||
|
@ -24,12 +24,15 @@ export const vidsrcembedScraper = makeEmbed({
|
||||||
if (!finalUrl.includes('.m3u8')) throw new Error('Unable to find HLS playlist');
|
if (!finalUrl.includes('.m3u8')) throw new Error('Unable to find HLS playlist');
|
||||||
|
|
||||||
return {
|
return {
|
||||||
stream: {
|
stream: [
|
||||||
type: 'hls',
|
{
|
||||||
playlist: finalUrl,
|
id: 'primary',
|
||||||
flags: [],
|
type: 'hls',
|
||||||
captions: [],
|
playlist: finalUrl,
|
||||||
},
|
flags: [flags.CORS_ALLOWED],
|
||||||
|
captions: [],
|
||||||
|
},
|
||||||
|
],
|
||||||
};
|
};
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
import { flags } from '@/main/targets';
|
import { flags } from '@/entrypoint/utils/targets';
|
||||||
import { makeSourcerer } from '@/providers/base';
|
import { makeSourcerer } from '@/providers/base';
|
||||||
import { scrapeMovie } from '@/providers/sources/vidsrc/scrape-movie';
|
import { scrapeMovie } from '@/providers/sources/vidsrc/scrape-movie';
|
||||||
import { scrapeShow } from '@/providers/sources/vidsrc/scrape-show';
|
import { scrapeShow } from '@/providers/sources/vidsrc/scrape-show';
|
||||||
|
@ -7,7 +7,7 @@ export const vidsrcScraper = makeSourcerer({
|
||||||
id: 'vidsrc',
|
id: 'vidsrc',
|
||||||
name: 'VidSrc',
|
name: 'VidSrc',
|
||||||
rank: 120,
|
rank: 120,
|
||||||
flags: [flags.NO_CORS],
|
flags: [flags.CORS_ALLOWED],
|
||||||
scrapeMovie,
|
scrapeMovie,
|
||||||
scrapeShow,
|
scrapeShow,
|
||||||
});
|
});
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
import { load } from 'cheerio';
|
import { load } from 'cheerio';
|
||||||
|
|
||||||
import { FetchReply } from '@/fetchers/fetch';
|
|
||||||
import { SourcererEmbed } from '@/providers/base';
|
import { SourcererEmbed } from '@/providers/base';
|
||||||
import { streambucketScraper } from '@/providers/embeds/streambucket';
|
import { streambucketScraper } from '@/providers/embeds/streambucket';
|
||||||
import { vidsrcembedScraper } from '@/providers/embeds/vidsrc';
|
import { vidsrcembedScraper } from '@/providers/embeds/vidsrc';
|
||||||
|
@ -44,7 +43,7 @@ async function getVidSrcEmbeds(ctx: MovieScrapeContext | ShowScrapeContext, star
|
||||||
html = await ctx.proxiedFetcher<string>(`/rcp/${hash}`, {
|
html = await ctx.proxiedFetcher<string>(`/rcp/${hash}`, {
|
||||||
baseUrl: vidsrcRCPBase,
|
baseUrl: vidsrcRCPBase,
|
||||||
headers: {
|
headers: {
|
||||||
referer: `${vidsrcBase}${startingURL}`,
|
referer: vidsrcBase,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -61,32 +60,23 @@ async function getVidSrcEmbeds(ctx: MovieScrapeContext | ShowScrapeContext, star
|
||||||
redirectURL = `https:${redirectURL}`;
|
redirectURL = `https:${redirectURL}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return the raw fetch response here.
|
const { finalUrl } = await ctx.proxiedFetcher.full(redirectURL, {
|
||||||
// When a Location header is sent, fetch
|
method: 'HEAD',
|
||||||
// will silently follow it. The "url" inside
|
|
||||||
// the Response is the final requested URL,
|
|
||||||
// which is the real embeds URL
|
|
||||||
const { url: embedURL } = await ctx.proxiedFetcher<FetchReply>(redirectURL, {
|
|
||||||
returnRaw: true,
|
|
||||||
method: 'HEAD', // We don't care about the actual response body here
|
|
||||||
headers: {
|
headers: {
|
||||||
referer: `${vidsrcRCPBase}/rcp/${hash}`,
|
referer: vidsrcBase,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
const embed: SourcererEmbed = {
|
const embed: SourcererEmbed = {
|
||||||
embedId: '',
|
embedId: '',
|
||||||
url: embedURL,
|
url: finalUrl,
|
||||||
};
|
};
|
||||||
|
|
||||||
const parsedUrl = new URL(embedURL);
|
const parsedUrl = new URL(finalUrl);
|
||||||
|
|
||||||
switch (parsedUrl.host) {
|
switch (parsedUrl.host) {
|
||||||
case 'vidsrc.stream':
|
case 'vidsrc.stream':
|
||||||
embed.embedId = vidsrcembedScraper.id;
|
embed.embedId = vidsrcembedScraper.id;
|
||||||
embed.headers = {
|
|
||||||
referer: `${vidsrcRCPBase}/rcp/${hash}`,
|
|
||||||
};
|
|
||||||
break;
|
break;
|
||||||
case 'streambucket.net':
|
case 'streambucket.net':
|
||||||
embed.embedId = streambucketScraper.id;
|
embed.embedId = streambucketScraper.id;
|
||||||
|
@ -99,7 +89,7 @@ async function getVidSrcEmbeds(ctx: MovieScrapeContext | ShowScrapeContext, star
|
||||||
// Just ignore this. This embed streams video over a custom WebSocket connection
|
// Just ignore this. This embed streams video over a custom WebSocket connection
|
||||||
break;
|
break;
|
||||||
default:
|
default:
|
||||||
throw new Error(`Failed to find VidSrc embed source for ${embedURL}`);
|
throw new Error(`Failed to find VidSrc embed source for ${finalUrl}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Since some embeds are ignored on purpose, check if a valid one was found
|
// Since some embeds are ignored on purpose, check if a valid one was found
|
||||||
|
|
|
@ -2,8 +2,14 @@ import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media';
|
||||||
import { UseableFetcher } from '@/fetchers/types';
|
import { UseableFetcher } from '@/fetchers/types';
|
||||||
|
|
||||||
export type ScrapeContext = {
|
export type ScrapeContext = {
|
||||||
proxiedFetcher: <T>(...params: Parameters<UseableFetcher<T>>) => ReturnType<UseableFetcher<T>>;
|
proxiedFetcher: {
|
||||||
fetcher: <T>(...params: Parameters<UseableFetcher<T>>) => ReturnType<UseableFetcher<T>>;
|
<T>(...params: Parameters<UseableFetcher<T>>): ReturnType<UseableFetcher<T>>;
|
||||||
|
full<T>(...params: Parameters<UseableFetcher<T>['full']>): ReturnType<UseableFetcher<T>['full']>;
|
||||||
|
};
|
||||||
|
fetcher: {
|
||||||
|
<T>(...params: Parameters<UseableFetcher<T>>): ReturnType<UseableFetcher<T>>;
|
||||||
|
full<T>(...params: Parameters<UseableFetcher<T>['full']>): ReturnType<UseableFetcher<T>['full']>;
|
||||||
|
};
|
||||||
progress(val: number): void;
|
progress(val: number): void;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue