Merge branch 'dev' into pr-14-v2

This commit is contained in:
Jorrin 2023-12-26 21:48:32 +01:00
commit 304ef68c5f
78 changed files with 2364 additions and 700 deletions

View File

@ -3,7 +3,7 @@ module.exports = {
browser: true,
},
extends: ['airbnb-base', 'plugin:@typescript-eslint/recommended', 'plugin:prettier/recommended'],
ignorePatterns: ['lib/*', 'tests/*', '/*.js', '/*.ts', '/**/*.test.ts', 'test/*'],
ignorePatterns: ['lib/*', 'tests/*', '/*.js', '/*.ts', '/src/__test__/*', '/**/*.test.ts', 'test/*'],
parser: '@typescript-eslint/parser',
parserOptions: {
project: './tsconfig.json',

View File

@ -10,7 +10,7 @@ features:
Visit documentation here: https://providers.docs.movie-web.app/
## Development
To make testing scrapers easier during development a CLI tool is available to run specific sources. To run the CLI testing tool, use `npm run test:dev`. The script supports 2 execution modes
To make testing scrapers easier during development a CLI tool is available to run specific sources. To run the CLI testing tool, use `npm run cli`. The script supports 2 execution modes
- CLI Mode, for passing in arguments directly to the script
- Question Mode, where the script asks you questions about which source you wish to test
@ -32,5 +32,5 @@ The following CLI Mode arguments are available
Example testing the FlixHQ source on the movie "Spirited Away"
```bash
npm run test:dev -- -sid flixhq -tid 129 -t movie
npm run cli -- -sid flixhq -tid 129 -t movie
```

1031
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -34,11 +34,11 @@
},
"homepage": "https://providers.docs.movie-web.app/",
"scripts": {
"build": "vite build",
"build": "vite build && tsc --noEmit",
"cli": "ts-node ./src/dev-cli/index.ts",
"test": "vitest run",
"test:dev": "ts-node ./src/dev-cli.ts",
"test:watch": "vitest",
"test:integration": "node ./tests/cjs && node ./tests/esm",
"test:integration": "node ./tests/cjs && node ./tests/esm && node ./tests/browser",
"test:coverage": "vitest run --coverage",
"lint": "eslint --ext .ts,.js src/",
"lint:fix": "eslint --fix --ext .ts,.js src/",
@ -65,6 +65,7 @@
"eslint-plugin-prettier": "^4.2.1",
"node-fetch": "^2.7.0",
"prettier": "^2.6.2",
"puppeteer": "^21.6.1",
"spinnies": "^0.5.1",
"ts-node": "^10.9.1",
"tsc-alias": "^1.6.7",

View File

@ -16,6 +16,8 @@ describe("makeSimpleProxyFetcher()", () => {
headers: new Headers({
"content-type": "text/plain",
}),
status: 204,
url: "test123",
text() {
return Promise.resolve(value);
},
@ -24,6 +26,8 @@ describe("makeSimpleProxyFetcher()", () => {
headers: new Headers({
"content-type": "application/json",
}),
status: 204,
url: "test123",
json() {
return Promise.resolve(value);
},
@ -31,7 +35,11 @@ describe("makeSimpleProxyFetcher()", () => {
}
function expectFetchCall(ops: { inputUrl: string, input: DefaultedFetcherOptions, outputUrl?: string, output: any, outputBody: any }) {
expect(fetcher(ops.inputUrl, ops.input)).resolves.toEqual(ops.outputBody);
const prom = fetcher(ops.inputUrl, ops.input);
expect((async () => (await prom).body)()).resolves.toEqual(ops.outputBody);
expect((async () => (await prom).headers.entries())()).resolves.toEqual((new Headers()).entries());
expect((async () => (await prom).statusCode)()).resolves.toEqual(204);
expect((async () => (await prom).finalUrl)()).resolves.toEqual("test123");
expect(fetch).toBeCalledWith(ops.outputUrl ?? ops.inputUrl, ops.output);
vi.clearAllMocks();
}
@ -43,6 +51,7 @@ describe("makeSimpleProxyFetcher()", () => {
input: {
method: "GET",
query: {},
readHeaders: [],
headers: {
"X-Hello": "world",
},
@ -62,6 +71,7 @@ describe("makeSimpleProxyFetcher()", () => {
input: {
method: "GET",
headers: {},
readHeaders: [],
query: {
"a": 'b',
}
@ -79,6 +89,7 @@ describe("makeSimpleProxyFetcher()", () => {
input: {
method: "GET",
query: {},
readHeaders: [],
headers: {},
},
outputUrl: `https://example.com/proxy?destination=${encodeURIComponent('https://google.com/')}`,
@ -97,6 +108,7 @@ describe("makeSimpleProxyFetcher()", () => {
input: {
method: "POST",
query: {},
readHeaders: [],
headers: {},
},
outputUrl: `https://example.com/proxy?destination=${encodeURIComponent('https://google.com/')}`,
@ -112,6 +124,7 @@ describe("makeSimpleProxyFetcher()", () => {
input: {
method: "POST",
query: {},
readHeaders: [],
headers: {},
},
outputUrl: `https://example.com/proxy?destination=${encodeURIComponent('https://google.com/')}`,

View File

@ -16,6 +16,8 @@ describe("makeStandardFetcher()", () => {
headers: new Headers({
"content-type": "text/plain",
}),
status: 204,
url: "test123",
text() {
return Promise.resolve(value);
},
@ -24,6 +26,8 @@ describe("makeStandardFetcher()", () => {
headers: new Headers({
"content-type": "application/json",
}),
status: 204,
url: "test123",
json() {
return Promise.resolve(value);
},
@ -31,7 +35,11 @@ describe("makeStandardFetcher()", () => {
}
function expectFetchCall(ops: { inputUrl: string, input: DefaultedFetcherOptions, outputUrl?: string, output: any, outputBody: any }) {
expect(fetcher(ops.inputUrl, ops.input)).resolves.toEqual(ops.outputBody);
const prom = fetcher(ops.inputUrl, ops.input);
expect((async () => (await prom).body)()).resolves.toEqual(ops.outputBody);
expect((async () => (await prom).headers.entries())()).resolves.toEqual((new Headers()).entries());
expect((async () => (await prom).statusCode)()).resolves.toEqual(204);
expect((async () => (await prom).finalUrl)()).resolves.toEqual("test123");
expect(fetch).toBeCalledWith(ops.outputUrl ?? ops.inputUrl, ops.output);
vi.clearAllMocks();
}
@ -43,6 +51,7 @@ describe("makeStandardFetcher()", () => {
input: {
method: "GET",
query: {},
readHeaders: [],
headers: {
"X-Hello": "world",
},
@ -53,6 +62,7 @@ describe("makeStandardFetcher()", () => {
headers: {
"X-Hello": "world",
},
body: undefined,
},
outputBody: "hello world"
})
@ -62,6 +72,7 @@ describe("makeStandardFetcher()", () => {
input: {
method: "GET",
headers: {},
readHeaders: [],
query: {
"a": 'b',
}
@ -79,6 +90,7 @@ describe("makeStandardFetcher()", () => {
input: {
query: {},
headers: {},
readHeaders: [],
method: "GET"
},
outputUrl: "https://google.com/",
@ -97,6 +109,7 @@ describe("makeStandardFetcher()", () => {
input: {
query: {},
headers: {},
readHeaders: [],
method: "POST"
},
outputUrl: "https://google.com/",
@ -112,6 +125,7 @@ describe("makeStandardFetcher()", () => {
input: {
query: {},
headers: {},
readHeaders: [],
method: "POST"
},
outputUrl: "https://google.com/",

View File

@ -15,40 +15,52 @@ export function makeProviderMocks() {
const sourceA = {
id: 'a',
name: 'A',
rank: 1,
disabled: false,
flags: [],
} as Sourcerer;
const sourceB = {
id: 'b',
name: 'B',
rank: 2,
disabled: false,
flags: [],
} as Sourcerer;
const sourceCDisabled = {
id: 'c',
name: 'C',
rank: 3,
disabled: true,
flags: [],
} as Sourcerer;
const sourceAHigherRank = {
id: 'a',
name: 'A',
rank: 100,
disabled: false,
flags: [],
} as Sourcerer;
const sourceGSameRankAsA = {
id: 'g',
name: 'G',
rank: 1,
disabled: false,
flags: [],
} as Sourcerer;
const fullSourceYMovie = {
id: 'y',
name: 'Y',
rank: 105,
scrapeMovie: vi.fn(),
flags: [],
} as Sourcerer;
const fullSourceYShow = {
id: 'y',
name: 'Y',
rank: 105,
scrapeShow: vi.fn(),
flags: [],
} as Sourcerer;
const fullSourceZBoth = {
id: 'z',
@ -56,6 +68,7 @@ const fullSourceZBoth = {
rank: 106,
scrapeMovie: vi.fn(),
scrapeShow: vi.fn(),
flags: [],
} as Sourcerer;
const embedD = {

View File

@ -1,12 +1,15 @@
import { mockEmbeds, mockSources } from '@/__test__/providerTests';
import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers';
import { FeatureMap } from '@/entrypoint/utils/targets';
import { getProviders } from '@/providers/get';
import { vi, describe, it, expect, afterEach } from 'vitest';
const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks());
const mocks = await vi.hoisted(async () => (await import('../providerTests')).makeProviderMocks());
vi.mock('@/providers/all', () => mocks);
const features = {
const features: FeatureMap = {
requires: [],
disallowed: []
}
describe('getProviders()', () => {
@ -17,7 +20,10 @@ describe('getProviders()', () => {
it('should return providers', () => {
mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD]);
mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]);
expect(getProviders(features)).toEqual({
expect(getProviders(features, {
embeds: getBuiltinEmbeds(),
sources: getBuiltinSources(),
})).toEqual({
sources: [mockSources.sourceA, mockSources.sourceB],
embeds: [mockEmbeds.embedD],
});
@ -26,7 +32,10 @@ describe('getProviders()', () => {
it('should filter out disabled providers', () => {
mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedEDisabled]);
mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceCDisabled, mockSources.sourceB]);
expect(getProviders(features)).toEqual({
expect(getProviders(features,{
embeds: getBuiltinEmbeds(),
sources: getBuiltinSources(),
})).toEqual({
sources: [mockSources.sourceA, mockSources.sourceB],
embeds: [mockEmbeds.embedD],
});
@ -35,31 +44,46 @@ describe('getProviders()', () => {
it('should throw on duplicate ids in sources', () => {
mocks.gatherAllEmbeds.mockReturnValue([]);
mocks.gatherAllSources.mockReturnValue([mockSources.sourceAHigherRank, mockSources.sourceA, mockSources.sourceB]);
expect(() => getProviders(features)).toThrowError();
expect(() => getProviders(features,{
embeds: getBuiltinEmbeds(),
sources: getBuiltinSources(),
})).toThrowError();
});
it('should throw on duplicate ids in embeds', () => {
mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedDHigherRank, mockEmbeds.embedA]);
mocks.gatherAllSources.mockReturnValue([]);
expect(() => getProviders(features)).toThrowError();
expect(() => getProviders(features,{
embeds: getBuiltinEmbeds(),
sources: getBuiltinSources(),
})).toThrowError();
});
it('should throw on duplicate ids between sources and embeds', () => {
mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedA]);
mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]);
expect(() => getProviders(features)).toThrowError();
expect(() => getProviders(features,{
embeds: getBuiltinEmbeds(),
sources: getBuiltinSources(),
})).toThrowError();
});
it('should throw on duplicate rank between sources and embeds', () => {
mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedA]);
mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]);
expect(() => getProviders(features)).toThrowError();
expect(() => getProviders(features,{
embeds: getBuiltinEmbeds(),
sources: getBuiltinSources(),
})).toThrowError();
});
it('should not throw with same rank between sources and embeds', () => {
mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedHSameRankAsSourceA]);
mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]);
expect(getProviders(features)).toEqual({
expect(getProviders(features,{
embeds: getBuiltinEmbeds(),
sources: getBuiltinSources(),
})).toEqual({
sources: [mockSources.sourceA, mockSources.sourceB],
embeds: [mockEmbeds.embedD, mockEmbeds.embedHSameRankAsSourceA],
});

View File

@ -1,6 +1,6 @@
import { mockEmbeds, mockSources } from '@/__test__/providerTests';
import { makeProviders } from '@/main/builder';
import { targets } from '@/main/targets.ts';
import { makeProviders } from '@/entrypoint/declare';
import { targets } from '@/entrypoint/utils/targets';
import { afterEach, describe, expect, it, vi } from 'vitest';
const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks());

View File

@ -1,6 +1,6 @@
import { mockEmbeds, mockSources } from '@/__test__/providerTests';
import { makeProviders } from '@/main/builder';
import { targets } from '@/main/targets.ts';
import { makeProviders } from '@/entrypoint/declare';
import { targets } from '@/entrypoint/utils/targets';
import { afterEach, describe, expect, it, vi } from 'vitest';
const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks());

View File

@ -0,0 +1,77 @@
import { FeatureMap, Flags, flags, flagsAllowedInFeatures } from "@/entrypoint/utils/targets";
import { describe, it, expect } from "vitest";
describe('flagsAllowedInFeatures()', () => {
function checkFeatures(featureMap: FeatureMap, flags: Flags[], output: boolean) {
expect(flagsAllowedInFeatures(featureMap, flags)).toEqual(output);
}
it('should check required correctly', () => {
checkFeatures({
requires: [],
disallowed: []
}, [], true);
checkFeatures({
requires: [flags.CORS_ALLOWED],
disallowed: []
}, [flags.CORS_ALLOWED], true);
checkFeatures({
requires: [flags.CORS_ALLOWED],
disallowed: []
}, [], false);
checkFeatures({
requires: [flags.CORS_ALLOWED, flags.IP_LOCKED],
disallowed: []
}, [flags.CORS_ALLOWED, flags.IP_LOCKED], true);
checkFeatures({
requires: [flags.IP_LOCKED],
disallowed: []
}, [flags.CORS_ALLOWED], false);
checkFeatures({
requires: [flags.IP_LOCKED],
disallowed: []
}, [], false);
});
it('should check disallowed correctly', () => {
checkFeatures({
requires: [],
disallowed: []
}, [], true);
checkFeatures({
requires: [],
disallowed: [flags.CORS_ALLOWED]
}, [], true);
checkFeatures({
requires: [],
disallowed: [flags.CORS_ALLOWED]
}, [flags.CORS_ALLOWED], false);
checkFeatures({
requires: [],
disallowed: [flags.CORS_ALLOWED]
}, [flags.IP_LOCKED], true);
checkFeatures({
requires: [],
disallowed: [flags.CORS_ALLOWED, flags.IP_LOCKED]
}, [flags.CORS_ALLOWED], false);
});
it('should pass mixed tests', () => {
checkFeatures({
requires: [flags.CORS_ALLOWED],
disallowed: [flags.IP_LOCKED]
}, [], false);
checkFeatures({
requires: [flags.CORS_ALLOWED],
disallowed: [flags.IP_LOCKED]
}, [flags.CORS_ALLOWED], true);
checkFeatures({
requires: [flags.CORS_ALLOWED],
disallowed: [flags.IP_LOCKED]
}, [flags.IP_LOCKED], false);
checkFeatures({
requires: [flags.CORS_ALLOWED],
disallowed: [flags.IP_LOCKED]
}, [flags.IP_LOCKED, flags.CORS_ALLOWED], false);
});
});

View File

@ -9,7 +9,9 @@ describe('isValidStream()', () => {
it('should pass valid streams', () => {
expect(isValidStream({
type: "file",
id: "a",
flags: [],
captions: [],
qualities: {
"1080": {
type: "mp4",
@ -19,7 +21,9 @@ describe('isValidStream()', () => {
})).toBe(true);
expect(isValidStream({
type: "hls",
id: "a",
flags: [],
captions: [],
playlist: "hello-world"
})).toBe(true);
});
@ -27,7 +31,9 @@ describe('isValidStream()', () => {
it('should detect empty qualities', () => {
expect(isValidStream({
type: "file",
id: "a",
flags: [],
captions: [],
qualities: {}
})).toBe(false);
});
@ -35,7 +41,9 @@ describe('isValidStream()', () => {
it('should detect empty stream urls', () => {
expect(isValidStream({
type: "file",
id: "a",
flags: [],
captions: [],
qualities: {
"1080": {
type: "mp4",
@ -48,7 +56,9 @@ describe('isValidStream()', () => {
it('should detect emtpy HLS playlists', () => {
expect(isValidStream({
type: "hls",
id: "a",
flags: [],
captions: [],
playlist: "",
})).toBe(false);
});

View File

@ -1,430 +0,0 @@
/* eslint import/no-extraneous-dependencies: ["error", {"devDependencies": true}] */
import util from 'node:util';
import { program } from 'commander';
import dotenv from 'dotenv';
import { prompt } from 'enquirer';
import nodeFetch from 'node-fetch';
import Spinnies from 'spinnies';
import { MetaOutput, MovieMedia, ProviderControls, ShowMedia, makeProviders, makeStandardFetcher, targets } from '.';
dotenv.config();
type ProviderSourceAnswers = {
id: string;
type: string;
};
type EmbedSourceAnswers = {
url: string;
};
type CommonAnswers = {
fetcher: string;
source: string;
};
type ShowAnswers = {
season: string;
episode: string;
};
type CommandLineArguments = {
fetcher: string;
sourceId: string;
tmdbId: string;
type: string;
season: string;
episode: string;
url: string;
headers?: Record<string, string>;
};
const TMDB_API_KEY = process.env.MOVIE_WEB_TMDB_API_KEY ?? '';
if (!TMDB_API_KEY?.trim()) {
throw new Error('Missing MOVIE_WEB_TMDB_API_KEY environment variable');
}
function logDeepObject(object: Record<any, any>) {
console.log(util.inspect(object, { showHidden: false, depth: null, colors: true }));
}
function getAllSources() {
// * The only way to get a list of all sources is to
// * create all these things. Maybe this should change
const providers = makeProviders({
fetcher: makeStandardFetcher(nodeFetch),
target: targets.NATIVE,
});
const combined = [...providers.listSources(), ...providers.listEmbeds()];
// * Remove dupes
const map = new Map(combined.map((source) => [source.id, source]));
return [...map.values()];
}
// * Defined here cuz ESLint didn't like the order these were defined in
const sources = getAllSources();
async function makeTMDBRequest(url: string): Promise<Response> {
const headers: {
accept: 'application/json';
authorization?: string;
} = {
accept: 'application/json',
};
// * Used to get around ESLint
// * Assignment to function parameter 'url'. eslint (no-param-reassign)
let requestURL = url;
// * JWT keys always start with ey and are ONLY valid as a header.
// * All other keys are ONLY valid as a query param.
// * Thanks TMDB.
if (TMDB_API_KEY.startsWith('ey')) {
headers.authorization = `Bearer ${TMDB_API_KEY}`;
} else {
requestURL += `?api_key=${TMDB_API_KEY}`;
}
return fetch(requestURL, {
method: 'GET',
headers,
});
}
async function getMovieMediaDetails(id: string): Promise<MovieMedia> {
const response = await makeTMDBRequest(`https://api.themoviedb.org/3/movie/${id}`);
const movie = await response.json();
if (movie.success === false) {
throw new Error(movie.status_message);
}
if (!movie.release_date) {
throw new Error(`${movie.title} has no release_date. Assuming unreleased`);
}
return {
type: 'movie',
title: movie.title,
releaseYear: Number(movie.release_date.split('-')[0]),
tmdbId: id,
};
}
async function getShowMediaDetails(id: string, seasonNumber: string, episodeNumber: string): Promise<ShowMedia> {
// * TV shows require the TMDB ID for the series, season, and episode
// * and the name of the series. Needs multiple requests
let response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}`);
const series = await response.json();
if (series.success === false) {
throw new Error(series.status_message);
}
if (!series.first_air_date) {
throw new Error(`${series.name} has no first_air_date. Assuming unaired`);
}
response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}`);
const season = await response.json();
if (season.success === false) {
throw new Error(season.status_message);
}
response = await makeTMDBRequest(
`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}/episode/${episodeNumber}`,
);
const episode = await response.json();
if (episode.success === false) {
throw new Error(episode.status_message);
}
return {
type: 'show',
title: series.name,
releaseYear: Number(series.first_air_date.split('-')[0]),
tmdbId: id,
episode: {
number: episode.episode_number,
tmdbId: episode.id,
},
season: {
number: season.season_number,
tmdbId: season.id,
},
};
}
function joinMediaTypes(mediaTypes: string[] | undefined) {
if (mediaTypes) {
const formatted = mediaTypes
.map((type: string) => {
return `${type[0].toUpperCase() + type.substring(1).toLowerCase()}s`;
})
.join(' / ');
return `(${formatted})`;
}
return ''; // * Embed sources pass through here too
}
async function runScraper(providers: ProviderControls, source: MetaOutput, options: CommandLineArguments) {
const spinnies = new Spinnies();
if (source.type === 'embed') {
spinnies.add('scrape', { text: `Running ${source.name} scraper on ${options.url}` });
try {
const result = await providers.runEmbedScraper({
url: options.url,
id: source.id,
headers: options.headers,
});
spinnies.succeed('scrape', { text: 'Done!' });
logDeepObject(result);
} catch (error) {
let message = 'Unknown error';
if (error instanceof Error) {
message = error.message;
}
spinnies.fail('scrape', { text: `ERROR: ${message}` });
}
} else {
let media;
if (options.type === 'movie') {
media = await getMovieMediaDetails(options.tmdbId);
} else {
media = await getShowMediaDetails(options.tmdbId, options.season, options.episode);
}
spinnies.add('scrape', { text: `Running ${source.name} scraper on ${media.title}` });
try {
const result = await providers.runSourceScraper({
media,
id: source.id,
});
spinnies.succeed('scrape', { text: 'Done!' });
logDeepObject(result);
} catch (error) {
let message = 'Unknown error';
if (error instanceof Error) {
message = error.message;
}
spinnies.fail('scrape', { text: `ERROR: ${message}` });
}
}
}
async function processOptions(options: CommandLineArguments) {
if (options.fetcher !== 'node-fetch' && options.fetcher !== 'native') {
throw new Error("Fetcher must be either 'native' or 'node-fetch'");
}
if (!options.sourceId.trim()) {
throw new Error('Source ID must be provided');
}
const source = sources.find(({ id }) => id === options.sourceId);
if (!source) {
throw new Error('Invalid source ID. No source found');
}
if (source.type === 'embed' && !options.url.trim()) {
throw new Error('Must provide an embed URL for embed sources');
}
if (source.type === 'source') {
if (!options.tmdbId.trim()) {
throw new Error('Must provide a TMDB ID for provider sources');
}
if (Number.isNaN(Number(options.tmdbId)) || Number(options.tmdbId) < 0) {
throw new Error('TMDB ID must be a number greater than 0');
}
if (!options.type.trim()) {
throw new Error('Must provide a type for provider sources');
}
if (options.type !== 'movie' && options.type !== 'show') {
throw new Error("Invalid media type. Must be either 'movie' or 'show'");
}
if (options.type === 'show') {
if (!options.season.trim()) {
throw new Error('Must provide a season number for TV shows');
}
if (!options.episode.trim()) {
throw new Error('Must provide an episode number for TV shows');
}
if (Number.isNaN(Number(options.season)) || Number(options.season) <= 0) {
throw new Error('Season number must be a number greater than 0');
}
if (Number.isNaN(Number(options.episode)) || Number(options.episode) <= 0) {
throw new Error('Episode number must be a number greater than 0');
}
}
}
if (typeof options.headers === 'string') {
options.headers = JSON.parse(options.headers);
}
let fetcher;
if (options.fetcher === 'native') {
fetcher = makeStandardFetcher(fetch);
} else {
fetcher = makeStandardFetcher(nodeFetch);
}
const providers = makeProviders({
fetcher,
target: targets.NATIVE,
});
await runScraper(providers, source, options);
}
async function runQuestions() {
const options = {
fetcher: 'node-fetch',
sourceId: '',
tmdbId: '',
type: 'movie',
season: '0',
episode: '0',
url: '',
};
const answers = await prompt<CommonAnswers>([
{
type: 'select',
name: 'fetcher',
message: 'Select a fetcher',
choices: [
{
message: 'Native',
name: 'native',
},
{
message: 'Node fetch',
name: 'node-fetch',
},
],
},
{
type: 'select',
name: 'source',
message: 'Select a source',
choices: sources.map((source) => ({
message: `[${source.type.toLocaleUpperCase()}] ${source.name} ${joinMediaTypes(source.mediaTypes)}`.trim(),
name: source.id,
})),
},
]);
options.fetcher = answers.fetcher;
options.sourceId = answers.source;
const source = sources.find(({ id }) => id === answers.source);
if (!source) {
throw new Error(`No source with ID ${answers.source} found`);
}
if (source.type === 'embed') {
const sourceAnswers = await prompt<EmbedSourceAnswers>([
{
type: 'input',
name: 'url',
message: 'Embed URL',
},
]);
options.url = sourceAnswers.url;
} else {
const sourceAnswers = await prompt<ProviderSourceAnswers>([
{
type: 'input',
name: 'id',
message: 'TMDB ID',
},
{
type: 'select',
name: 'type',
message: 'Media type',
choices: [
{
message: 'Movie',
name: 'movie',
},
{
message: 'TV Show',
name: 'show',
},
],
},
]);
options.tmdbId = sourceAnswers.id;
options.type = sourceAnswers.type;
if (sourceAnswers.type === 'show') {
const seriesAnswers = await prompt<ShowAnswers>([
{
type: 'input',
name: 'season',
message: 'Season',
},
{
type: 'input',
name: 'episode',
message: 'Episode',
},
]);
options.season = seriesAnswers.season;
options.episode = seriesAnswers.episode;
}
}
await processOptions(options);
}
async function runCommandLine() {
program
.option('-f, --fetcher <fetcher>', "Fetcher to use. Either 'native' or 'node-fetch'", 'node-fetch')
.option('-sid, --source-id <id>', 'ID for the source to use. Either an embed or provider', '')
.option('-tid, --tmdb-id <id>', 'TMDB ID for the media to scrape. Only used if source is a provider', '')
.option('-t, --type <type>', "Media type. Either 'movie' or 'show'. Only used if source is a provider", 'movie')
.option('-s, --season <number>', "Season number. Only used if type is 'show'", '0')
.option('-e, --episode <number>', "Episode number. Only used if type is 'show'", '0')
.option('-u, --url <embed URL>', 'URL to a video embed. Only used if source is an embed', '')
.option('-h, --headers <JSON>', 'Optional headers to pass to scrapers. JSON encoded');
program.parse();
await processOptions(program.opts());
}
if (process.argv.length === 2) {
runQuestions();
} else {
runCommandLine();
}

1
src/dev-cli/browser/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
dist

View File

@ -0,0 +1,11 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Scraper CLI</title>
</head>
<body>
<script src="./index.ts" type="module"></script>
</body>
</html>

View File

@ -0,0 +1,17 @@
import { makeProviders, makeSimpleProxyFetcher, makeStandardFetcher, targets } from '../../../lib';
(window as any).scrape = (proxyUrl: string, type: 'source' | 'embed', input: any) => {
const providers = makeProviders({
fetcher: makeStandardFetcher(fetch),
target: targets.BROWSER,
proxiedFetcher: makeSimpleProxyFetcher(proxyUrl, fetch),
});
if (type === 'source') {
return providers.runSourceScraper(input);
}
if (type === 'embed') {
return providers.runEmbedScraper(input);
}
throw new Error('Input input type');
};

16
src/dev-cli/config.ts Normal file
View File

@ -0,0 +1,16 @@
export function getConfig() {
let tmdbApiKey = process.env.MOVIE_WEB_TMDB_API_KEY ?? '';
tmdbApiKey = tmdbApiKey.trim();
if (!tmdbApiKey) {
throw new Error('Missing MOVIE_WEB_TMDB_API_KEY environment variable');
}
let proxyUrl: undefined | string = process.env.MOVIE_WEB_PROXY_URL;
proxyUrl = !proxyUrl ? undefined : proxyUrl;
return {
tmdbApiKey,
proxyUrl,
};
}

185
src/dev-cli/index.ts Normal file
View File

@ -0,0 +1,185 @@
/* eslint import/no-extraneous-dependencies: ["error", {"devDependencies": true}] */
import { program } from 'commander';
import dotenv from 'dotenv';
import { prompt } from 'enquirer';
import { runScraper } from '@/dev-cli/scraper';
import { processOptions } from '@/dev-cli/validate';
import { getBuiltinEmbeds, getBuiltinSources } from '..';
dotenv.config();
type ProviderSourceAnswers = {
id: string;
type: string;
};
type EmbedSourceAnswers = {
url: string;
};
type CommonAnswers = {
fetcher: string;
source: string;
};
type ShowAnswers = {
season: string;
episode: string;
};
const sourceScrapers = getBuiltinSources().sort((a, b) => b.rank - a.rank);
const embedScrapers = getBuiltinEmbeds().sort((a, b) => b.rank - a.rank);
const sources = [...sourceScrapers, ...embedScrapers];
function joinMediaTypes(mediaTypes: string[] | undefined) {
if (mediaTypes) {
const formatted = mediaTypes
.map((type: string) => {
return `${type[0].toUpperCase() + type.substring(1).toLowerCase()}s`;
})
.join(' / ');
return `(${formatted})`;
}
return ''; // * Embed sources pass through here too
}
async function runQuestions() {
const options = {
fetcher: 'node-fetch',
sourceId: '',
tmdbId: '',
type: 'movie',
season: '0',
episode: '0',
url: '',
};
const answers = await prompt<CommonAnswers>([
{
type: 'select',
name: 'fetcher',
message: 'Select a fetcher mode',
choices: [
{
message: 'Native',
name: 'native',
},
{
message: 'Node fetch',
name: 'node-fetch',
},
{
message: 'Browser',
name: 'browser',
},
],
},
{
type: 'select',
name: 'source',
message: 'Select a source',
choices: sources.map((source) => ({
message: `[${source.type.toLocaleUpperCase()}] ${source.name} ${joinMediaTypes(source.mediaTypes)}`.trim(),
name: source.id,
})),
},
]);
options.fetcher = answers.fetcher;
options.sourceId = answers.source;
const source = sources.find(({ id }) => id === answers.source);
if (!source) {
throw new Error(`No source with ID ${answers.source} found`);
}
if (source.type === 'embed') {
const sourceAnswers = await prompt<EmbedSourceAnswers>([
{
type: 'input',
name: 'url',
message: 'Embed URL',
},
]);
options.url = sourceAnswers.url;
} else {
const sourceAnswers = await prompt<ProviderSourceAnswers>([
{
type: 'input',
name: 'id',
message: 'TMDB ID',
},
{
type: 'select',
name: 'type',
message: 'Media type',
choices: [
{
message: 'Movie',
name: 'movie',
},
{
message: 'TV Show',
name: 'show',
},
],
},
]);
options.tmdbId = sourceAnswers.id;
options.type = sourceAnswers.type;
if (sourceAnswers.type === 'show') {
const seriesAnswers = await prompt<ShowAnswers>([
{
type: 'input',
name: 'season',
message: 'Season',
},
{
type: 'input',
name: 'episode',
message: 'Episode',
},
]);
options.season = seriesAnswers.season;
options.episode = seriesAnswers.episode;
}
}
const { providerOptions, source: validatedSource, options: validatedOps } = await processOptions(sources, options);
await runScraper(providerOptions, validatedSource, validatedOps);
}
async function runCommandLine() {
program
.option('-f, --fetcher <fetcher>', "Fetcher to use. Either 'native' or 'node-fetch'", 'node-fetch')
.option('-sid, --source-id <id>', 'ID for the source to use. Either an embed or provider', '')
.option('-tid, --tmdb-id <id>', 'TMDB ID for the media to scrape. Only used if source is a provider', '')
.option('-t, --type <type>', "Media type. Either 'movie' or 'show'. Only used if source is a provider", 'movie')
.option('-s, --season <number>', "Season number. Only used if type is 'show'", '0')
.option('-e, --episode <number>', "Episode number. Only used if type is 'show'", '0')
.option('-u, --url <embed URL>', 'URL to a video embed. Only used if source is an embed', '');
program.parse();
const {
providerOptions,
source: validatedSource,
options: validatedOps,
} = await processOptions(sources, program.opts());
await runScraper(providerOptions, validatedSource, validatedOps);
}
if (process.argv.length === 2) {
runQuestions().catch(() => console.error('Exited.'));
} else {
runCommandLine().catch(() => console.error('Exited.'));
}

5
src/dev-cli/logging.ts Normal file
View File

@ -0,0 +1,5 @@
import { inspect } from 'node:util';
export function logDeepObject(object: Record<any, any>) {
console.log(inspect(object, { showHidden: false, depth: null, colors: true }));
}

136
src/dev-cli/scraper.ts Normal file
View File

@ -0,0 +1,136 @@
/* eslint import/no-extraneous-dependencies: ["error", {"devDependencies": true}] */
import { existsSync } from 'fs';
import { join } from 'path';
import puppeteer, { Browser } from 'puppeteer';
import Spinnies from 'spinnies';
import { PreviewServer, build, preview } from 'vite';
import { getConfig } from '@/dev-cli/config';
import { logDeepObject } from '@/dev-cli/logging';
import { getMovieMediaDetails, getShowMediaDetails } from '@/dev-cli/tmdb';
import { CommandLineArguments } from '@/dev-cli/validate';
import { MetaOutput, ProviderMakerOptions, makeProviders } from '..';
async function runBrowserScraping(
providerOptions: ProviderMakerOptions,
source: MetaOutput,
options: CommandLineArguments,
) {
if (!existsSync(join(__dirname, '../../lib/index.mjs')))
throw new Error('Please compile before running cli in browser mode');
const config = getConfig();
if (!config.proxyUrl)
throw new Error('Simple proxy url must be set in the environment (MOVIE_WEB_PROXY_URL) for browser mode to work');
const root = join(__dirname, 'browser');
let server: PreviewServer | undefined;
let browser: Browser | undefined;
try {
// setup browser
await build({
root,
});
server = await preview({
root,
});
browser = await puppeteer.launch({
headless: 'new',
args: ['--no-sandbox', '--disable-setuid-sandbox'],
});
const page = await browser.newPage();
await page.goto(server.resolvedUrls.local[0]);
await page.waitForFunction('!!window.scrape', { timeout: 5000 });
// get input media
let input: any;
if (source.type === 'embed') {
input = {
url: options.url,
id: source.id,
};
} else if (source.type === 'source') {
let media;
if (options.type === 'movie') {
media = await getMovieMediaDetails(options.tmdbId);
} else {
media = await getShowMediaDetails(options.tmdbId, options.season, options.episode);
}
input = {
media,
id: source.id,
};
} else {
throw new Error('Wrong source input type');
}
return await page.evaluate(
async (proxy, type, inp) => {
return (window as any).scrape(proxy, type, inp);
},
config.proxyUrl,
source.type,
input,
);
} finally {
server?.httpServer.close();
await browser?.close();
}
}
async function runActualScraping(
providerOptions: ProviderMakerOptions,
source: MetaOutput,
options: CommandLineArguments,
): Promise<any> {
if (options.fetcher === 'browser') return runBrowserScraping(providerOptions, source, options);
const providers = makeProviders(providerOptions);
if (source.type === 'embed') {
return providers.runEmbedScraper({
url: options.url,
id: source.id,
});
}
if (source.type === 'source') {
let media;
if (options.type === 'movie') {
media = await getMovieMediaDetails(options.tmdbId);
} else {
media = await getShowMediaDetails(options.tmdbId, options.season, options.episode);
}
return providers.runSourceScraper({
media,
id: source.id,
});
}
throw new Error('Invalid source type');
}
export async function runScraper(
providerOptions: ProviderMakerOptions,
source: MetaOutput,
options: CommandLineArguments,
) {
const spinnies = new Spinnies();
spinnies.add('scrape', { text: `Running ${source.name} scraper` });
try {
const result = await runActualScraping(providerOptions, source, options);
spinnies.succeed('scrape', { text: 'Done!' });
logDeepObject(result);
} catch (error) {
let message = 'Unknown error';
if (error instanceof Error) {
message = error.message;
}
spinnies.fail('scrape', { text: `ERROR: ${message}` });
console.error(error);
}
}

95
src/dev-cli/tmdb.ts Normal file
View File

@ -0,0 +1,95 @@
import { getConfig } from '@/dev-cli/config';
import { MovieMedia, ShowMedia } from '..';
export async function makeTMDBRequest(url: string): Promise<Response> {
const headers: {
accept: 'application/json';
authorization?: string;
} = {
accept: 'application/json',
};
let requestURL = url;
const key = getConfig().tmdbApiKey;
// * JWT keys always start with ey and are ONLY valid as a header.
// * All other keys are ONLY valid as a query param.
// * Thanks TMDB.
if (key.startsWith('ey')) {
headers.authorization = `Bearer ${key}`;
} else {
requestURL += `?api_key=${key}`;
}
return fetch(requestURL, {
method: 'GET',
headers,
});
}
export async function getMovieMediaDetails(id: string): Promise<MovieMedia> {
const response = await makeTMDBRequest(`https://api.themoviedb.org/3/movie/${id}`);
const movie = await response.json();
if (movie.success === false) {
throw new Error(movie.status_message);
}
if (!movie.release_date) {
throw new Error(`${movie.title} has no release_date. Assuming unreleased`);
}
return {
type: 'movie',
title: movie.title,
releaseYear: Number(movie.release_date.split('-')[0]),
tmdbId: id,
};
}
export async function getShowMediaDetails(id: string, seasonNumber: string, episodeNumber: string): Promise<ShowMedia> {
// * TV shows require the TMDB ID for the series, season, and episode
// * and the name of the series. Needs multiple requests
let response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}`);
const series = await response.json();
if (series.success === false) {
throw new Error(series.status_message);
}
if (!series.first_air_date) {
throw new Error(`${series.name} has no first_air_date. Assuming unaired`);
}
response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}`);
const season = await response.json();
if (season.success === false) {
throw new Error(season.status_message);
}
response = await makeTMDBRequest(
`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}/episode/${episodeNumber}`,
);
const episode = await response.json();
if (episode.success === false) {
throw new Error(episode.status_message);
}
return {
type: 'show',
title: series.name,
releaseYear: Number(series.first_air_date.split('-')[0]),
tmdbId: id,
episode: {
number: episode.episode_number,
tmdbId: episode.id,
},
season: {
number: season.season_number,
tmdbId: season.id,
},
};
}

91
src/dev-cli/validate.ts Normal file
View File

@ -0,0 +1,91 @@
import nodeFetch from 'node-fetch';
import { Embed, Sourcerer } from '@/providers/base';
import { ProviderMakerOptions, makeStandardFetcher, targets } from '..';
export type CommandLineArguments = {
fetcher: string;
sourceId: string;
tmdbId: string;
type: string;
season: string;
episode: string;
url: string;
};
export async function processOptions(sources: Array<Embed | Sourcerer>, options: CommandLineArguments) {
const fetcherOptions = ['node-fetch', 'native', 'browser'];
if (!fetcherOptions.includes(options.fetcher)) {
throw new Error(`Fetcher must be any of: ${fetcherOptions.join()}`);
}
if (!options.sourceId.trim()) {
throw new Error('Source ID must be provided');
}
const source = sources.find(({ id }) => id === options.sourceId);
if (!source) {
throw new Error('Invalid source ID. No source found');
}
if (source.type === 'embed' && !options.url.trim()) {
throw new Error('Must provide an embed URL for embed sources');
}
if (source.type === 'source') {
if (!options.tmdbId.trim()) {
throw new Error('Must provide a TMDB ID for provider sources');
}
if (Number.isNaN(Number(options.tmdbId)) || Number(options.tmdbId) < 0) {
throw new Error('TMDB ID must be a number greater than 0');
}
if (!options.type.trim()) {
throw new Error('Must provide a type for provider sources');
}
if (options.type !== 'movie' && options.type !== 'show') {
throw new Error("Invalid media type. Must be either 'movie' or 'show'");
}
if (options.type === 'show') {
if (!options.season.trim()) {
throw new Error('Must provide a season number for TV shows');
}
if (!options.episode.trim()) {
throw new Error('Must provide an episode number for TV shows');
}
if (Number.isNaN(Number(options.season)) || Number(options.season) <= 0) {
throw new Error('Season number must be a number greater than 0');
}
if (Number.isNaN(Number(options.episode)) || Number(options.episode) <= 0) {
throw new Error('Episode number must be a number greater than 0');
}
}
}
let fetcher;
if (options.fetcher === 'native') {
fetcher = makeStandardFetcher(fetch);
} else {
fetcher = makeStandardFetcher(nodeFetch);
}
const providerOptions: ProviderMakerOptions = {
fetcher,
target: targets.ANY,
};
return {
providerOptions,
options,
source,
};
}

93
src/entrypoint/builder.ts Normal file
View File

@ -0,0 +1,93 @@
import { ProviderControls, makeControls } from '@/entrypoint/controls';
import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers';
import { Targets, getTargetFeatures } from '@/entrypoint/utils/targets';
import { Fetcher } from '@/fetchers/types';
import { Embed, Sourcerer } from '@/providers/base';
import { getProviders } from '@/providers/get';
export type ProviderBuilder = {
setTarget(target: Targets): ProviderBuilder;
setFetcher(fetcher: Fetcher): ProviderBuilder;
setProxiedFetcher(fetcher: Fetcher): ProviderBuilder;
addSource(scraper: Sourcerer): ProviderBuilder;
addSource(name: string): ProviderBuilder;
addEmbed(scraper: Embed): ProviderBuilder;
addEmbed(name: string): ProviderBuilder;
addBuiltinProviders(): ProviderBuilder;
enableConsistentIpForRequests(): ProviderBuilder;
build(): ProviderControls;
};
export function buildProviders(): ProviderBuilder {
let consistentIpForRequests = false;
let target: Targets | null = null;
let fetcher: Fetcher | null = null;
let proxiedFetcher: Fetcher | null = null;
const embeds: Embed[] = [];
const sources: Sourcerer[] = [];
const builtinSources = getBuiltinSources();
const builtinEmbeds = getBuiltinEmbeds();
return {
enableConsistentIpForRequests() {
consistentIpForRequests = true;
return this;
},
setFetcher(f) {
fetcher = f;
return this;
},
setProxiedFetcher(f) {
proxiedFetcher = f;
return this;
},
setTarget(t) {
target = t;
return this;
},
addSource(input) {
if (typeof input !== 'string') {
sources.push(input);
return this;
}
const matchingSource = builtinSources.find((v) => v.id === input);
if (!matchingSource) throw new Error('Source not found');
sources.push(matchingSource);
return this;
},
addEmbed(input) {
if (typeof input !== 'string') {
embeds.push(input);
return this;
}
const matchingEmbed = builtinEmbeds.find((v) => v.id === input);
if (!matchingEmbed) throw new Error('Embed not found');
embeds.push(matchingEmbed);
return this;
},
addBuiltinProviders() {
sources.push(...builtinSources);
embeds.push(...builtinEmbeds);
return this;
},
build() {
if (!target) throw new Error('Target not set');
if (!fetcher) throw new Error('Fetcher not set');
const features = getTargetFeatures(target, consistentIpForRequests);
const list = getProviders(features, {
embeds,
sources,
});
return makeControls({
fetcher,
proxiedFetcher: proxiedFetcher ?? undefined,
embeds: list.embeds,
sources: list.sources,
features,
});
},
};
}

View File

@ -1,24 +1,19 @@
import { makeFullFetcher } from '@/fetchers/common';
import { FullScraperEvents, IndividualScraperEvents } from '@/entrypoint/utils/events';
import { ScrapeMedia } from '@/entrypoint/utils/media';
import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/entrypoint/utils/meta';
import { FeatureMap } from '@/entrypoint/utils/targets';
import { makeFetcher } from '@/fetchers/common';
import { Fetcher } from '@/fetchers/types';
import { FullScraperEvents, IndividualScraperEvents } from '@/main/events';
import { scrapeIndividualEmbed, scrapeInvidualSource } from '@/main/individualRunner';
import { ScrapeMedia } from '@/main/media';
import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/main/meta';
import { RunOutput, runAllProviders } from '@/main/runner';
import { Targets, getTargetFeatures } from '@/main/targets';
import { EmbedOutput, SourcererOutput } from '@/providers/base';
import { getProviders } from '@/providers/get';
import { Embed, EmbedOutput, Sourcerer, SourcererOutput } from '@/providers/base';
import { scrapeIndividualEmbed, scrapeInvidualSource } from '@/runners/individualRunner';
import { RunOutput, runAllProviders } from '@/runners/runner';
export interface ProviderBuilderOptions {
// fetcher, every web request gets called through here
export interface ProviderControlsInput {
fetcher: Fetcher;
// proxied fetcher, if the scraper needs to access a CORS proxy. this fetcher will be called instead
// of the normal fetcher. Defaults to the normal fetcher.
proxiedFetcher?: Fetcher;
// target of where the streams will be used
target: Targets;
features: FeatureMap;
sources: Sourcerer[];
embeds: Embed[];
}
export interface RunnerOptions {
@ -83,13 +78,16 @@ export interface ProviderControls {
listEmbeds(): MetaOutput[];
}
export function makeProviders(ops: ProviderBuilderOptions): ProviderControls {
const features = getTargetFeatures(ops.target);
const list = getProviders(features);
export function makeControls(ops: ProviderControlsInput): ProviderControls {
const list = {
embeds: ops.embeds,
sources: ops.sources,
};
const providerRunnerOps = {
features,
fetcher: makeFullFetcher(ops.fetcher),
proxiedFetcher: makeFullFetcher(ops.proxiedFetcher ?? ops.fetcher),
features: ops.features,
fetcher: makeFetcher(ops.fetcher),
proxiedFetcher: makeFetcher(ops.proxiedFetcher ?? ops.fetcher),
};
return {

37
src/entrypoint/declare.ts Normal file
View File

@ -0,0 +1,37 @@
import { makeControls } from '@/entrypoint/controls';
import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers';
import { Targets, getTargetFeatures } from '@/entrypoint/utils/targets';
import { Fetcher } from '@/fetchers/types';
import { getProviders } from '@/providers/get';
export interface ProviderMakerOptions {
// fetcher, every web request gets called through here
fetcher: Fetcher;
// proxied fetcher, if the scraper needs to access a CORS proxy. this fetcher will be called instead
// of the normal fetcher. Defaults to the normal fetcher.
proxiedFetcher?: Fetcher;
// target of where the streams will be used
target: Targets;
// Set this to true, if the requests will have the same IP as
// the device that the stream will be played on
consistentIpForRequests?: boolean;
}
export function makeProviders(ops: ProviderMakerOptions) {
const features = getTargetFeatures(ops.target, ops.consistentIpForRequests ?? false);
const list = getProviders(features, {
embeds: getBuiltinEmbeds(),
sources: getBuiltinSources(),
});
return makeControls({
embeds: list.embeds,
sources: list.sources,
features,
fetcher: ops.fetcher,
proxiedFetcher: ops.proxiedFetcher,
});
}

View File

@ -0,0 +1,10 @@
import { gatherAllEmbeds, gatherAllSources } from '@/providers/all';
import { Embed, Sourcerer } from '@/providers/base';
export function getBuiltinSources(): Sourcerer[] {
return gatherAllSources();
}
export function getBuiltinEmbeds(): Embed[] {
return gatherAllEmbeds();
}

View File

@ -1,4 +1,4 @@
import { MediaTypes } from '@/main/media';
import { MediaTypes } from '@/entrypoint/utils/media';
import { Embed, Sourcerer } from '@/providers/base';
import { ProviderList } from '@/providers/get';

View File

@ -0,0 +1,64 @@
export const flags = {
// CORS are set to allow any origin
CORS_ALLOWED: 'cors-allowed',
// the stream is locked on IP, so only works if
// request maker is same as player (not compatible with proxies)
IP_LOCKED: 'ip-locked',
} as const;
export type Flags = (typeof flags)[keyof typeof flags];
export const targets = {
// browser with CORS restrictions
BROWSER: 'browser',
// browser, but no CORS restrictions through a browser extension
BROWSER_EXTENSION: 'browser-extension',
// native app, so no restrictions in what can be played
NATIVE: 'native',
// any target, no target restrictions
ANY: 'any',
} as const;
export type Targets = (typeof targets)[keyof typeof targets];
export type FeatureMap = {
requires: Flags[];
disallowed: Flags[];
};
export const targetToFeatures: Record<Targets, FeatureMap> = {
browser: {
requires: [flags.CORS_ALLOWED],
disallowed: [],
},
'browser-extension': {
requires: [],
disallowed: [],
},
native: {
requires: [],
disallowed: [],
},
any: {
requires: [],
disallowed: [],
},
};
export function getTargetFeatures(target: Targets, consistentIpForRequests: boolean): FeatureMap {
const features = targetToFeatures[target];
if (!consistentIpForRequests) features.disallowed.push(flags.IP_LOCKED);
return features;
}
export function flagsAllowedInFeatures(features: FeatureMap, inputFlags: Flags[]): boolean {
const hasAllFlags = features.requires.every((v) => inputFlags.includes(v));
if (!hasAllFlags) return false;
const hasDisallowedFlag = features.disallowed.some((v) => inputFlags.includes(v));
if (hasDisallowedFlag) return false;
return true;
}

View File

@ -26,15 +26,19 @@ export function makeFullUrl(url: string, ops?: FullUrlOptions): string {
return parsedUrl.toString();
}
export function makeFullFetcher(fetcher: Fetcher): UseableFetcher {
return (url, ops) => {
export function makeFetcher(fetcher: Fetcher): UseableFetcher {
const newFetcher = (url: string, ops?: FetcherOptions) => {
return fetcher(url, {
headers: ops?.headers ?? {},
method: ops?.method ?? 'GET',
query: ops?.query ?? {},
baseUrl: ops?.baseUrl ?? '',
readHeaders: ops?.readHeaders ?? [],
body: ops?.body,
returnRaw: ops?.returnRaw ?? false,
});
};
const output: UseableFetcher = async (url, ops) => (await newFetcher(url, ops)).body;
output.full = newFetcher;
return output;
}

View File

@ -11,13 +11,17 @@ export type FetchOps = {
export type FetchHeaders = {
get(key: string): string | null;
set(key: string, value: string): void;
};
export type FetchReply = {
text(): Promise<string>;
json(): Promise<any>;
extraHeaders?: FetchHeaders;
extraUrl?: string;
headers: FetchHeaders;
url: string;
status: number;
};
export type FetchLike = (url: string, ops?: FetchOps | undefined) => Promise<FetchReply>;

View File

@ -9,9 +9,28 @@ const headerMap: Record<string, string> = {
origin: 'X-Origin',
};
const responseHeaderMap: Record<string, string> = {
'x-set-cookie': 'Set-Cookie',
};
export function makeSimpleProxyFetcher(proxyUrl: string, f: FetchLike): Fetcher {
const fetcher = makeStandardFetcher(f);
const proxiedFetch: Fetcher = async (url, ops) => {
const fetcher = makeStandardFetcher(async (a, b) => {
const res = await f(a, b);
// set extra headers that cant normally be accessed
res.extraHeaders = new Headers();
Object.entries(responseHeaderMap).forEach((entry) => {
const value = res.headers.get(entry[0]);
if (!value) return;
res.extraHeaders?.set(entry[0].toLowerCase(), value);
});
// set correct final url
res.extraUrl = res.headers.get('X-Final-Destination') ?? res.url;
return res;
});
const fullUrl = makeFullUrl(url, ops);
const headerEntries = Object.entries(ops.headers).map((entry) => {

View File

@ -1,8 +1,20 @@
import { serializeBody } from '@/fetchers/body';
import { makeFullUrl } from '@/fetchers/common';
import { FetchLike } from '@/fetchers/fetch';
import { FetchLike, FetchReply } from '@/fetchers/fetch';
import { Fetcher } from '@/fetchers/types';
function getHeaders(list: string[], res: FetchReply): Headers {
const output = new Headers();
list.forEach((header) => {
const realHeader = header.toLowerCase();
const value = res.headers.get(realHeader);
const extraValue = res.extraHeaders?.get(realHeader);
if (!value) return;
output.set(realHeader, extraValue ?? value);
});
return output;
}
export function makeStandardFetcher(f: FetchLike): Fetcher {
const normalFetch: Fetcher = async (url, ops) => {
const fullUrl = makeFullUrl(url, ops);
@ -21,9 +33,17 @@ export function makeStandardFetcher(f: FetchLike): Fetcher {
return res;
}
let body: any;
const isJson = res.headers.get('content-type')?.includes('application/json');
if (isJson) return res.json();
return res.text();
if (isJson) body = await res.json();
else body = await res.text();
return {
body,
finalUrl: res.extraUrl ?? res.url,
headers: getHeaders(ops.readHeaders, res),
statusCode: res.status,
};
};
return normalFetch;

View File

@ -5,10 +5,13 @@ export type FetcherOptions = {
headers?: Record<string, string>;
query?: Record<string, string>;
method?: 'HEAD' | 'GET' | 'POST';
readHeaders?: string[];
body?: Record<string, any> | string | FormData | URLSearchParams;
returnRaw?: boolean;
};
// Version of the options that always has the defaults set
// This is to make making fetchers yourself easier
export type DefaultedFetcherOptions = {
baseUrl?: string;
body?: Record<string, any> | string | FormData;
@ -16,13 +19,23 @@ export type DefaultedFetcherOptions = {
query: Record<string, string>;
method: 'HEAD' | 'GET' | 'POST';
returnRaw: boolean;
readHeaders: string[];
};
export type FetcherResponse<T = any> = {
statusCode: number;
headers: Headers;
finalUrl: string;
body: T;
};
// This is the version that will be inputted by library users
export type Fetcher<T = any> = {
(url: string, ops: DefaultedFetcherOptions): Promise<T>;
(url: string, ops: DefaultedFetcherOptions): Promise<FetcherResponse<T>>;
};
// this feature has some quality of life features
// This is the version that scrapers will be interacting with
export type UseableFetcher<T = any> = {
(url: string, ops?: FetcherOptions): Promise<T>;
full: (url: string, ops?: FetcherOptions) => Promise<FetcherResponse<T>>;
};

View File

@ -1,19 +1,19 @@
export type { EmbedOutput, SourcererOutput } from '@/providers/base';
export type { RunOutput } from '@/main/runner';
export type { MetaOutput } from '@/main/meta';
export type { FullScraperEvents } from '@/main/events';
export type { Targets, Flags } from '@/main/targets';
export type { MediaTypes, ShowMedia, ScrapeMedia, MovieMedia } from '@/main/media';
export type {
ProviderBuilderOptions,
ProviderControls,
RunnerOptions,
EmbedRunnerOptions,
SourceRunnerOptions,
} from '@/main/builder';
export type { Stream, StreamFile, FileBasedStream, HlsBasedStream, Qualities } from '@/providers/streams';
export type { Fetcher, FetcherOptions, FetcherResponse } from '@/fetchers/types';
export type { RunOutput } from '@/runners/runner';
export type { MetaOutput } from '@/entrypoint/utils/meta';
export type { FullScraperEvents } from '@/entrypoint/utils/events';
export type { Targets, Flags } from '@/entrypoint/utils/targets';
export type { MediaTypes, ShowMedia, ScrapeMedia, MovieMedia } from '@/entrypoint/utils/media';
export type { ProviderControls, RunnerOptions, EmbedRunnerOptions, SourceRunnerOptions } from '@/entrypoint/controls';
export type { ProviderBuilder } from '@/entrypoint/builder';
export type { ProviderMakerOptions } from '@/entrypoint/declare';
export { NotFoundError } from '@/utils/errors';
export { makeProviders } from '@/main/builder';
export { makeProviders } from '@/entrypoint/declare';
export { buildProviders } from '@/entrypoint/builder';
export { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers';
export { makeStandardFetcher } from '@/fetchers/standardFetch';
export { makeSimpleProxyFetcher } from '@/fetchers/simpleProxy';
export { flags, targets } from '@/main/targets';
export { flags, targets } from '@/entrypoint/utils/targets';

View File

@ -1,40 +0,0 @@
export const flags = {
NO_CORS: 'no-cors',
IP_LOCKED: 'ip-locked',
} as const;
export type Flags = (typeof flags)[keyof typeof flags];
export const targets = {
BROWSER: 'browser',
NATIVE: 'native',
ALL: 'all',
} as const;
export type Targets = (typeof targets)[keyof typeof targets];
export type FeatureMap = {
requires: readonly Flags[];
};
export const targetToFeatures: Record<Targets, FeatureMap> = {
browser: {
requires: [flags.NO_CORS],
},
native: {
requires: [],
},
all: {
requires: [],
},
} as const;
export function getTargetFeatures(target: Targets): FeatureMap {
return targetToFeatures[target];
}
export function flagsAllowedInFeatures(features: FeatureMap, inputFlags: Flags[]): boolean {
const hasAllFlags = features.requires.every((v) => inputFlags.includes(v));
if (!hasAllFlags) return false;
return true;
}

View File

@ -1,7 +1,9 @@
import { Flags } from '@/main/targets';
import { Flags } from '@/entrypoint/utils/targets';
import { Stream } from '@/providers/streams';
import { EmbedScrapeContext, MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
export type MediaScraperTypes = 'show' | 'movie';
export type SourcererEmbed = {
embedId: string;
url: string;
@ -10,10 +12,10 @@ export type SourcererEmbed = {
export type SourcererOutput = {
embeds: SourcererEmbed[];
stream?: Stream;
stream?: Stream[];
};
export type Sourcerer = {
export type SourcererOptions = {
id: string;
name: string; // displayed in the UI
rank: number; // the higher the number, the earlier it gets put on the queue
@ -23,15 +25,29 @@ export type Sourcerer = {
scrapeShow?: (input: ShowScrapeContext) => Promise<SourcererOutput>;
};
export function makeSourcerer(state: Sourcerer): Sourcerer {
return state;
export type Sourcerer = SourcererOptions & {
type: 'source';
disabled: boolean;
mediaTypes: MediaScraperTypes[];
};
export function makeSourcerer(state: SourcererOptions): Sourcerer {
const mediaTypes: MediaScraperTypes[] = [];
if (state.scrapeMovie) mediaTypes.push('movie');
if (state.scrapeShow) mediaTypes.push('show');
return {
...state,
type: 'source',
disabled: state.disabled ?? false,
mediaTypes,
};
}
export type EmbedOutput = {
stream: Stream;
stream: Stream[];
};
export type Embed = {
export type EmbedOptions = {
id: string;
name: string; // displayed in the UI
rank: number; // the higher the number, the earlier it gets put on the queue
@ -39,6 +55,17 @@ export type Embed = {
scrape: (input: EmbedScrapeContext) => Promise<EmbedOutput>;
};
export function makeEmbed(state: Embed): Embed {
return state;
export type Embed = EmbedOptions & {
type: 'embed';
disabled: boolean;
mediaTypes: undefined;
};
export function makeEmbed(state: EmbedOptions): Embed {
return {
...state,
type: 'embed',
disabled: state.disabled ?? false,
mediaTypes: undefined,
};
}

View File

@ -8,6 +8,7 @@ export type CaptionType = keyof typeof captionTypes;
export type Caption = {
type: CaptionType;
id: string; // only unique per stream
url: string;
hasCorsRestrictions: boolean;
language: string;

View File

@ -1,4 +1,4 @@
import { MediaTypes } from '@/main/media';
import { MediaTypes } from '@/entrypoint/utils/media';
export const febBoxBase = `https://www.febbox.com`;

View File

@ -1,4 +1,4 @@
import { MediaTypes } from '@/main/media';
import { MediaTypes } from '@/entrypoint/utils/media';
import { FebboxFileList, febBoxBase } from '@/providers/embeds/febbox/common';
import { EmbedScrapeContext } from '@/utils/context';

View File

@ -1,5 +1,5 @@
import { MediaTypes } from '@/main/media';
import { flags } from '@/main/targets';
import { MediaTypes } from '@/entrypoint/utils/media';
import { flags } from '@/entrypoint/utils/targets';
import { makeEmbed } from '@/providers/base';
import { parseInputUrl } from '@/providers/embeds/febbox/common';
import { getStreams } from '@/providers/embeds/febbox/fileList';
@ -36,12 +36,15 @@ export const febboxHlsScraper = makeEmbed({
ctx.progress(70);
return {
stream: {
type: 'hls',
flags: [flags.NO_CORS],
captions: await getSubtitles(ctx, id, firstStream.fid, type as MediaTypes, season, episode),
playlist: `https://www.febbox.com/hls/main/${firstStream.oss_fid}.m3u8`,
},
stream: [
{
id: 'primary',
type: 'hls',
flags: [flags.CORS_ALLOWED],
captions: await getSubtitles(ctx, id, firstStream.fid, type as MediaTypes, season, episode),
playlist: `https://www.febbox.com/hls/main/${firstStream.oss_fid}.m3u8`,
},
],
};
},
});

View File

@ -1,4 +1,4 @@
import { flags } from '@/main/targets';
import { flags } from '@/entrypoint/utils/targets';
import { makeEmbed } from '@/providers/base';
import { parseInputUrl } from '@/providers/embeds/febbox/common';
import { getStreamQualities } from '@/providers/embeds/febbox/qualities';
@ -39,12 +39,15 @@ export const febboxMp4Scraper = makeEmbed({
ctx.progress(70);
return {
stream: {
captions: await getSubtitles(ctx, id, fid, type, episode, season),
qualities,
type: 'file',
flags: [flags.NO_CORS],
},
stream: [
{
id: 'primary',
captions: await getSubtitles(ctx, id, fid, type, episode, season),
qualities,
type: 'file',
flags: [flags.CORS_ALLOWED],
},
],
};
},
});

View File

@ -54,6 +54,7 @@ export async function getSubtitles(
if (!validCode) return;
output.push({
id: subtitleFilePath,
language: subtitle.lang,
hasCorsRestrictions: true,
type: subtitleType,

View File

@ -33,21 +33,24 @@ export const mixdropScraper = makeEmbed({
const url = link[1];
return {
stream: {
type: 'file',
flags: [],
captions: [],
qualities: {
unknown: {
type: 'mp4',
url: url.startsWith('http') ? url : `https:${url}`, // URLs don't always start with the protocol
headers: {
// MixDrop requires this header on all streams
Referer: 'https://mixdrop.co/',
stream: [
{
id: 'primary',
type: 'file',
flags: [],
captions: [],
qualities: {
unknown: {
type: 'mp4',
url: url.startsWith('http') ? url : `https:${url}`, // URLs don't always start with the protocol
headers: {
// MixDrop requires this header on all streams
Referer: 'https://mixdrop.co/',
},
},
},
},
},
],
};
},
});

View File

@ -1,4 +1,4 @@
import { flags } from '@/main/targets';
import { flags } from '@/entrypoint/utils/targets';
import { makeEmbed } from '@/providers/base';
export const mp4uploadScraper = makeEmbed({
@ -15,17 +15,20 @@ export const mp4uploadScraper = makeEmbed({
if (!streamUrl) throw new Error('Stream url not found in embed code');
return {
stream: {
type: 'file',
flags: [flags.NO_CORS],
captions: [],
qualities: {
'1080': {
type: 'mp4',
url: streamUrl,
stream: [
{
id: 'primary',
type: 'file',
flags: [flags.CORS_ALLOWED],
captions: [],
qualities: {
'1080': {
type: 'mp4',
url: streamUrl,
},
},
},
},
],
};
},
});

View File

@ -1,6 +1,6 @@
import { load } from 'cheerio';
import { flags } from '@/main/targets';
import { flags } from '@/entrypoint/utils/targets';
import { makeEmbed } from '@/providers/base';
type DPlayerSourcesResponse = {
@ -57,12 +57,15 @@ export const smashyStreamDScraper = makeEmbed({
);
return {
stream: {
playlist: playlistRes,
type: 'hls',
flags: [flags.NO_CORS],
captions: [],
},
stream: [
{
id: 'primary',
playlist: playlistRes,
type: 'hls',
flags: [flags.CORS_ALLOWED],
captions: [],
},
],
};
},
});

View File

@ -1,4 +1,4 @@
import { flags } from '@/main/targets';
import { flags } from '@/entrypoint/utils/targets';
import { makeEmbed } from '@/providers/base';
import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions';
@ -30,6 +30,7 @@ export const smashyStreamFScraper = makeEmbed({
const captionType = getCaptionTypeFromUrl(url);
if (!languageCode || !captionType) return null;
return {
id: url,
url: url.replace(',', ''),
language: languageCode,
type: captionType,
@ -42,12 +43,15 @@ export const smashyStreamFScraper = makeEmbed({
.filter((x): x is Caption => x !== null) ?? [];
return {
stream: {
playlist: res.sourceUrls[0],
type: 'hls',
flags: [flags.NO_CORS],
captions,
},
stream: [
{
id: 'primary',
playlist: res.sourceUrls[0],
type: 'hls',
flags: [flags.CORS_ALLOWED],
captions,
},
],
};
},
});

View File

@ -3,7 +3,7 @@ import Base64 from 'crypto-js/enc-base64';
import Utf8 from 'crypto-js/enc-utf8';
import FormData from 'form-data';
import { flags } from '@/main/targets';
import { flags } from '@/entrypoint/utils/targets';
import { makeEmbed } from '@/providers/base';
import { StreamFile } from '@/providers/streams';
import { EmbedScrapeContext } from '@/utils/context';
@ -155,12 +155,15 @@ export const streamsbScraper = makeEmbed({
}, {} as Record<string, StreamFile>);
return {
stream: {
type: 'file',
flags: [flags.NO_CORS],
qualities,
captions: [],
},
stream: [
{
id: 'primary',
type: 'file',
flags: [flags.CORS_ALLOWED],
qualities,
captions: [],
},
],
};
},
});

View File

@ -1,6 +1,6 @@
import crypto from 'crypto-js';
import { flags } from '@/main/targets';
import { flags } from '@/entrypoint/utils/targets';
import { makeEmbed } from '@/providers/base';
import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions';
@ -110,6 +110,7 @@ export const upcloudScraper = makeEmbed({
const language = labelToLanguageCode(track.label);
if (!language) return;
captions.push({
id: track.file,
language,
hasCorsRestrictions: false,
type,
@ -118,12 +119,15 @@ export const upcloudScraper = makeEmbed({
});
return {
stream: {
type: 'hls',
playlist: sources.file,
flags: [flags.NO_CORS],
captions,
},
stream: [
{
id: 'primary',
type: 'hls',
playlist: sources.file,
flags: [flags.CORS_ALLOWED],
captions,
},
],
};
},
});

View File

@ -1,6 +1,6 @@
import * as unpacker from 'unpacker';
import { flags } from '@/main/targets';
import { flags } from '@/entrypoint/utils/targets';
import { makeEmbed } from '@/providers/base';
const packedRegex = /(eval\(function\(p,a,c,k,e,d\).*\)\)\))/;
@ -21,12 +21,15 @@ export const upstreamScraper = makeEmbed({
if (link) {
return {
stream: {
type: 'hls',
playlist: link[1],
flags: [flags.NO_CORS],
captions: [],
},
stream: [
{
id: 'primary',
type: 'hls',
playlist: link[1],
flags: [flags.CORS_ALLOWED],
captions: [],
},
],
};
}
}

View File

@ -1,5 +1,4 @@
import { FeatureMap, flagsAllowedInFeatures } from '@/main/targets';
import { gatherAllEmbeds, gatherAllSources } from '@/providers/all';
import { FeatureMap, flagsAllowedInFeatures } from '@/entrypoint/utils/targets';
import { Embed, Sourcerer } from '@/providers/base';
import { hasDuplicates } from '@/utils/predicates';
@ -8,9 +7,9 @@ export interface ProviderList {
embeds: Embed[];
}
export function getProviders(features: FeatureMap): ProviderList {
const sources = gatherAllSources().filter((v) => !v?.disabled);
const embeds = gatherAllEmbeds().filter((v) => !v?.disabled);
export function getProviders(features: FeatureMap, list: ProviderList): ProviderList {
const sources = list.sources.filter((v) => !v?.disabled);
const embeds = list.embeds.filter((v) => !v?.disabled);
const combined = [...sources, ...embeds];
const anyDuplicateId = hasDuplicates(combined.map((v) => v.id));

View File

@ -1,4 +1,4 @@
import { flags } from '@/main/targets';
import { flags } from '@/entrypoint/utils/targets';
import { makeSourcerer } from '@/providers/base';
import { upcloudScraper } from '@/providers/embeds/upcloud';
import { getFlixhqMovieSources, getFlixhqShowSources, getFlixhqSourceDetails } from '@/providers/sources/flixhq/scrape';
@ -9,7 +9,7 @@ export const flixhqScraper = makeSourcerer({
id: 'flixhq',
name: 'FlixHQ',
rank: 100,
flags: [flags.NO_CORS],
flags: [flags.CORS_ALLOWED],
async scrapeMovie(ctx) {
const id = await getFlixhqId(ctx, ctx.media);
if (!id) throw new NotFoundError('no search results match');

View File

@ -1,6 +1,6 @@
import { load } from 'cheerio';
import { MovieMedia, ShowMedia } from '@/main/media';
import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media';
import { flixHqBase } from '@/providers/sources/flixhq/common';
import { ScrapeContext } from '@/utils/context';
import { NotFoundError } from '@/utils/errors';

View File

@ -1,6 +1,6 @@
import { load } from 'cheerio';
import { MovieMedia, ShowMedia } from '@/main/media';
import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media';
import { flixHqBase } from '@/providers/sources/flixhq/common';
import { compareMedia, compareTitle } from '@/utils/compare';
import { ScrapeContext } from '@/utils/context';

View File

@ -1,6 +1,6 @@
import { load } from 'cheerio';
import { flags } from '@/main/targets';
import { flags } from '@/entrypoint/utils/targets';
import { makeSourcerer } from '@/providers/base';
import { upcloudScraper } from '@/providers/embeds/upcloud';
import { NotFoundError } from '@/utils/errors';
@ -13,7 +13,7 @@ export const goMoviesScraper = makeSourcerer({
id: 'gomovies',
name: 'GOmovies',
rank: 110,
flags: [flags.NO_CORS],
flags: [flags.CORS_ALLOWED],
async scrapeShow(ctx) {
const search = await ctx.proxiedFetcher<string>(`/ajax/search`, {
method: 'POST',

View File

@ -1,6 +1,6 @@
import { load } from 'cheerio';
import { flags } from '@/main/targets';
import { flags } from '@/entrypoint/utils/targets';
import { makeSourcerer } from '@/providers/base';
import { NotFoundError } from '@/utils/errors';
@ -13,7 +13,7 @@ export const kissAsianScraper = makeSourcerer({
id: 'kissasian',
name: 'KissAsian',
rank: 130,
flags: [flags.NO_CORS],
flags: [flags.CORS_ALLOWED],
disabled: true,
async scrapeShow(ctx) {

View File

@ -1,4 +1,4 @@
import { flags } from '@/main/targets';
import { flags } from '@/entrypoint/utils/targets';
import { SourcererOutput, makeSourcerer } from '@/providers/base';
import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
import { NotFoundError } from '@/utils/errors';
@ -17,12 +17,15 @@ async function universalScraper(ctx: MovieScrapeContext | ShowScrapeContext): Pr
return {
embeds: [],
stream: {
playlist: videoUrl,
type: 'hls',
flags: [flags.IP_LOCKED],
captions: [],
},
stream: [
{
id: 'primary',
playlist: videoUrl,
type: 'hls',
flags: [flags.IP_LOCKED],
captions: [],
},
],
};
}

View File

@ -1,4 +1,4 @@
import { MovieMedia } from '@/main/media';
import { MovieMedia } from '@/entrypoint/utils/media';
// ! Types
interface BaseConfig {

View File

@ -1,4 +1,4 @@
import { MovieMedia, ShowMedia } from '@/main/media';
import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media';
import { compareMedia } from '@/utils/compare';
import { ScrapeContext } from '@/utils/context';
import { NotFoundError } from '@/utils/errors';

View File

@ -1,4 +1,4 @@
import { MovieMedia, ShowMedia } from '@/main/media';
import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media';
import { ScrapeContext } from '@/utils/context';
import { StreamsDataResult } from './type';

View File

@ -1,4 +1,4 @@
import { flags } from '@/main/targets';
import { flags } from '@/entrypoint/utils/targets';
import { makeSourcerer } from '@/providers/base';
import { NotFoundError } from '@/utils/errors';
@ -8,7 +8,7 @@ export const remotestreamScraper = makeSourcerer({
id: 'remotestream',
name: 'Remote Stream',
rank: 55,
flags: [flags.NO_CORS],
flags: [flags.CORS_ALLOWED],
async scrapeShow(ctx) {
const seasonNumber = ctx.media.season.number;
const episodeNumber = ctx.media.episode.number;
@ -22,12 +22,15 @@ export const remotestreamScraper = makeSourcerer({
return {
embeds: [],
stream: {
captions: [],
playlist: playlistLink,
type: 'hls',
flags: [flags.NO_CORS],
},
stream: [
{
id: 'primary',
captions: [],
playlist: playlistLink,
type: 'hls',
flags: [flags.CORS_ALLOWED],
},
],
};
},
async scrapeMovie(ctx) {
@ -40,12 +43,15 @@ export const remotestreamScraper = makeSourcerer({
return {
embeds: [],
stream: {
captions: [],
playlist: playlistLink,
type: 'hls',
flags: [flags.NO_CORS],
},
stream: [
{
id: 'primary',
captions: [],
playlist: playlistLink,
type: 'hls',
flags: [flags.CORS_ALLOWED],
},
],
};
},
});

View File

@ -1,4 +1,4 @@
import { flags } from '@/main/targets';
import { flags } from '@/entrypoint/utils/targets';
import { SourcererOutput, makeSourcerer } from '@/providers/base';
import { febboxHlsScraper } from '@/providers/embeds/febbox/hls';
import { febboxMp4Scraper } from '@/providers/embeds/febbox/mp4';
@ -47,7 +47,7 @@ export const showboxScraper = makeSourcerer({
id: 'showbox',
name: 'Showbox',
rank: 300,
flags: [flags.NO_CORS],
flags: [flags.CORS_ALLOWED],
scrapeShow: comboScraper,
scrapeMovie: comboScraper,
});

View File

@ -1,6 +1,6 @@
import { load } from 'cheerio';
import { flags } from '@/main/targets';
import { flags } from '@/entrypoint/utils/targets';
import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base';
import { smashyStreamDScraper } from '@/providers/embeds/smashystream/dued';
import { smashyStreamFScraper } from '@/providers/embeds/smashystream/video1';
@ -58,7 +58,7 @@ export const smashyStreamScraper = makeSourcerer({
id: 'smashystream',
name: 'SmashyStream',
rank: 70,
flags: [flags.NO_CORS],
flags: [flags.CORS_ALLOWED],
scrapeMovie: universalScraper,
scrapeShow: universalScraper,
});

View File

@ -1,4 +1,4 @@
import { flags } from '@/main/targets';
import { flags } from '@/entrypoint/utils/targets';
import { makeSourcerer } from '@/providers/base';
import { scrapeMovie } from '@/providers/sources/zoechip/scrape-movie';
import { scrapeShow } from '@/providers/sources/zoechip/scrape-show';
@ -7,7 +7,7 @@ export const zoechipScraper = makeSourcerer({
id: 'zoechip',
name: 'ZoeChip',
rank: 200,
flags: [flags.NO_CORS],
flags: [flags.CORS_ALLOWED],
scrapeMovie,
scrapeShow,
});

View File

@ -1,6 +1,6 @@
import { load } from 'cheerio';
import { ShowMedia } from '@/main/media';
import { ShowMedia } from '@/entrypoint/utils/media';
import { ZoeChipSourceDetails, zoeBase } from '@/providers/sources/zoechip/common';
import { MovieScrapeContext, ScrapeContext, ShowScrapeContext } from '@/utils/context';

View File

@ -1,6 +1,6 @@
import { load } from 'cheerio';
import { MovieMedia, ShowMedia } from '@/main/media';
import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media';
import { zoeBase } from '@/providers/sources/zoechip/common';
import { compareMedia } from '@/utils/compare';
import { ScrapeContext } from '@/utils/context';

View File

@ -1,4 +1,4 @@
import { Flags } from '@/main/targets';
import { Flags } from '@/entrypoint/utils/targets';
import { Caption } from '@/providers/captions';
export type StreamFile = {
@ -9,18 +9,22 @@ export type StreamFile = {
export type Qualities = 'unknown' | '360' | '480' | '720' | '1080' | '4k';
export type FileBasedStream = {
type: 'file';
type StreamCommon = {
id: string; // only unique per output
flags: Flags[];
qualities: Partial<Record<Qualities, StreamFile>>;
captions: Caption[];
headers?: Record<string, string>; // these headers HAVE to be set to watch the stream
preferredHeaders?: Record<string, string>; // these headers are optional, would improve the stream
};
export type HlsBasedStream = {
export type FileBasedStream = StreamCommon & {
type: 'file';
qualities: Partial<Record<Qualities, StreamFile>>;
};
export type HlsBasedStream = StreamCommon & {
type: 'hls';
flags: Flags[];
playlist: string;
captions: Caption[];
};
export type Stream = FileBasedStream | HlsBasedStream;

View File

@ -1,7 +1,7 @@
import { IndividualScraperEvents } from '@/entrypoint/utils/events';
import { ScrapeMedia } from '@/entrypoint/utils/media';
import { FeatureMap, flagsAllowedInFeatures } from '@/entrypoint/utils/targets';
import { UseableFetcher } from '@/fetchers/types';
import { IndividualScraperEvents } from '@/main/events';
import { ScrapeMedia } from '@/main/media';
import { FeatureMap, flagsAllowedInFeatures } from '@/main/targets';
import { EmbedOutput, SourcererOutput } from '@/providers/base';
import { ProviderList } from '@/providers/get';
import { ScrapeContext } from '@/utils/context';
@ -50,12 +50,16 @@ export async function scrapeInvidualSource(
media: ops.media,
});
// stream doesn't satisfy the feature flags, so gets removed in output
if (output?.stream && (!isValidStream(output.stream) || !flagsAllowedInFeatures(ops.features, output.stream.flags))) {
output.stream = undefined;
// filter output with only valid streams
if (output?.stream) {
output.stream = output.stream
.filter((stream) => isValidStream(stream))
.filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
}
if (!output) throw new Error('output is null');
if ((!output.stream || output.stream.length === 0) && output.embeds.length === 0)
throw new NotFoundError('No streams found');
return output;
}
@ -90,9 +94,10 @@ export async function scrapeIndividualEmbed(
},
});
if (!isValidStream(output.stream)) throw new NotFoundError('stream is incomplete');
if (!flagsAllowedInFeatures(ops.features, output.stream.flags))
throw new NotFoundError("stream doesn't satisfy target feature flags");
output.stream = output.stream
.filter((stream) => isValidStream(stream))
.filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
if (output.stream.length === 0) throw new NotFoundError('No streams found');
return output;
}

View File

@ -1,7 +1,7 @@
import { FullScraperEvents } from '@/entrypoint/utils/events';
import { ScrapeMedia } from '@/entrypoint/utils/media';
import { FeatureMap, flagsAllowedInFeatures } from '@/entrypoint/utils/targets';
import { UseableFetcher } from '@/fetchers/types';
import { FullScraperEvents } from '@/main/events';
import { ScrapeMedia } from '@/main/media';
import { FeatureMap, flagsAllowedInFeatures } from '@/main/targets';
import { EmbedOutput, SourcererOutput } from '@/providers/base';
import { ProviderList } from '@/providers/get';
import { Stream } from '@/providers/streams';
@ -18,13 +18,13 @@ export type RunOutput = {
export type SourceRunOutput = {
sourceId: string;
stream?: Stream;
stream: Stream[];
embeds: [];
};
export type EmbedRunOutput = {
embedId: string;
stream?: Stream;
stream: Stream[];
};
export type ProviderRunnerOptions = {
@ -80,12 +80,14 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
...contextBase,
media: ops.media,
});
if (output?.stream && !isValidStream(output?.stream)) {
throw new NotFoundError('stream is incomplete');
}
if (output?.stream && !flagsAllowedInFeatures(ops.features, output.stream.flags)) {
throw new NotFoundError("stream doesn't satisfy target feature flags");
if (output) {
output.stream = (output.stream ?? [])
.filter((stream) => isValidStream(stream))
.filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
}
if (!output) throw Error('No output');
if ((!output.stream || output.stream.length === 0) && output.embeds.length === 0)
throw new NotFoundError('No streams found');
} catch (err) {
if (err instanceof NotFoundError) {
ops.events?.update?.({
@ -107,10 +109,10 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
if (!output) throw new Error('Invalid media type');
// return stream is there are any
if (output.stream) {
if (output.stream?.[0]) {
return {
sourceId: s.id,
stream: output.stream,
stream: output.stream[0],
};
}
@ -144,9 +146,10 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
...contextBase,
url: e.url,
});
if (!flagsAllowedInFeatures(ops.features, embedOutput.stream.flags)) {
throw new NotFoundError("stream doesn't satisfy target feature flags");
}
embedOutput.stream = embedOutput.stream
.filter((stream) => isValidStream(stream))
.filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
if (embedOutput.stream.length === 0) throw new NotFoundError('No streams found');
} catch (err) {
if (err instanceof NotFoundError) {
ops.events?.update?.({
@ -169,7 +172,7 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
return {
sourceId: s.id,
embedId: scraper.id,
stream: embedOutput.stream,
stream: embedOutput.stream[0],
};
}
}

View File

@ -1,4 +1,4 @@
import { CommonMedia } from '@/main/media';
import { CommonMedia } from '@/entrypoint/utils/media';
export function normalizeTitle(title: string): string {
return title

View File

@ -1,5 +1,5 @@
import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media';
import { UseableFetcher } from '@/fetchers/types';
import { MovieMedia, ShowMedia } from '@/main/media';
export type ScrapeContext = {
proxiedFetcher: <T>(...params: Parameters<UseableFetcher<T>>) => ReturnType<UseableFetcher<T>>;

1
tests/browser/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
dist

11
tests/browser/index.html Normal file
View File

@ -0,0 +1,11 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Browser integration test</title>
</head>
<body>
<script type="module" src="index.ts"></script>
</body>
</html>

8
tests/browser/index.ts Normal file
View File

@ -0,0 +1,8 @@
import { makeProviders, makeStandardFetcher, targets } from '../../lib/index.mjs';
(window as any).TEST = () => {
makeProviders({
fetcher: makeStandardFetcher(fetch),
target: targets.ANY,
});
}

View File

@ -0,0 +1,3 @@
{
"main": "startup.mjs"
}

31
tests/browser/startup.mjs Normal file
View File

@ -0,0 +1,31 @@
import { build, preview } from 'vite';
import puppeteer from 'puppeteer';
import { fileURLToPath } from 'url';
import { dirname } from 'path';
const root = dirname(fileURLToPath(import.meta.url));
await build({
root,
});
const server = await preview({
root,
});
let browser;
try {
browser = await puppeteer.launch({
headless: 'new',
args: ['--no-sandbox', '--disable-setuid-sandbox'],
});
const page = await browser.newPage();
await page.goto(server.resolvedUrls.local[0]);
await page.waitForFunction('!!window.TEST', { timeout: 5000 });
await page.evaluate(() => {
window.TEST();
});
} finally {
server.httpServer.close();
await browser.close();
}
console.log('Success!');

View File

@ -17,7 +17,7 @@
}
},
"include": ["src"],
"exclude": ["node_modules", "**/__tests__/*"],
"exclude": ["node_modules", "**/__test__"],
"ts-node": {
"require": ["tsconfig-paths/register"]
}