Merge branch 'dev' into pr-14-v2
This commit is contained in:
commit
304ef68c5f
|
@ -3,7 +3,7 @@ module.exports = {
|
|||
browser: true,
|
||||
},
|
||||
extends: ['airbnb-base', 'plugin:@typescript-eslint/recommended', 'plugin:prettier/recommended'],
|
||||
ignorePatterns: ['lib/*', 'tests/*', '/*.js', '/*.ts', '/**/*.test.ts', 'test/*'],
|
||||
ignorePatterns: ['lib/*', 'tests/*', '/*.js', '/*.ts', '/src/__test__/*', '/**/*.test.ts', 'test/*'],
|
||||
parser: '@typescript-eslint/parser',
|
||||
parserOptions: {
|
||||
project: './tsconfig.json',
|
||||
|
|
|
@ -10,7 +10,7 @@ features:
|
|||
Visit documentation here: https://providers.docs.movie-web.app/
|
||||
|
||||
## Development
|
||||
To make testing scrapers easier during development a CLI tool is available to run specific sources. To run the CLI testing tool, use `npm run test:dev`. The script supports 2 execution modes
|
||||
To make testing scrapers easier during development a CLI tool is available to run specific sources. To run the CLI testing tool, use `npm run cli`. The script supports 2 execution modes
|
||||
|
||||
- CLI Mode, for passing in arguments directly to the script
|
||||
- Question Mode, where the script asks you questions about which source you wish to test
|
||||
|
@ -32,5 +32,5 @@ The following CLI Mode arguments are available
|
|||
Example testing the FlixHQ source on the movie "Spirited Away"
|
||||
|
||||
```bash
|
||||
npm run test:dev -- -sid flixhq -tid 129 -t movie
|
||||
npm run cli -- -sid flixhq -tid 129 -t movie
|
||||
```
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -34,11 +34,11 @@
|
|||
},
|
||||
"homepage": "https://providers.docs.movie-web.app/",
|
||||
"scripts": {
|
||||
"build": "vite build",
|
||||
"build": "vite build && tsc --noEmit",
|
||||
"cli": "ts-node ./src/dev-cli/index.ts",
|
||||
"test": "vitest run",
|
||||
"test:dev": "ts-node ./src/dev-cli.ts",
|
||||
"test:watch": "vitest",
|
||||
"test:integration": "node ./tests/cjs && node ./tests/esm",
|
||||
"test:integration": "node ./tests/cjs && node ./tests/esm && node ./tests/browser",
|
||||
"test:coverage": "vitest run --coverage",
|
||||
"lint": "eslint --ext .ts,.js src/",
|
||||
"lint:fix": "eslint --fix --ext .ts,.js src/",
|
||||
|
@ -65,6 +65,7 @@
|
|||
"eslint-plugin-prettier": "^4.2.1",
|
||||
"node-fetch": "^2.7.0",
|
||||
"prettier": "^2.6.2",
|
||||
"puppeteer": "^21.6.1",
|
||||
"spinnies": "^0.5.1",
|
||||
"ts-node": "^10.9.1",
|
||||
"tsc-alias": "^1.6.7",
|
||||
|
|
|
@ -16,6 +16,8 @@ describe("makeSimpleProxyFetcher()", () => {
|
|||
headers: new Headers({
|
||||
"content-type": "text/plain",
|
||||
}),
|
||||
status: 204,
|
||||
url: "test123",
|
||||
text() {
|
||||
return Promise.resolve(value);
|
||||
},
|
||||
|
@ -24,6 +26,8 @@ describe("makeSimpleProxyFetcher()", () => {
|
|||
headers: new Headers({
|
||||
"content-type": "application/json",
|
||||
}),
|
||||
status: 204,
|
||||
url: "test123",
|
||||
json() {
|
||||
return Promise.resolve(value);
|
||||
},
|
||||
|
@ -31,7 +35,11 @@ describe("makeSimpleProxyFetcher()", () => {
|
|||
}
|
||||
|
||||
function expectFetchCall(ops: { inputUrl: string, input: DefaultedFetcherOptions, outputUrl?: string, output: any, outputBody: any }) {
|
||||
expect(fetcher(ops.inputUrl, ops.input)).resolves.toEqual(ops.outputBody);
|
||||
const prom = fetcher(ops.inputUrl, ops.input);
|
||||
expect((async () => (await prom).body)()).resolves.toEqual(ops.outputBody);
|
||||
expect((async () => (await prom).headers.entries())()).resolves.toEqual((new Headers()).entries());
|
||||
expect((async () => (await prom).statusCode)()).resolves.toEqual(204);
|
||||
expect((async () => (await prom).finalUrl)()).resolves.toEqual("test123");
|
||||
expect(fetch).toBeCalledWith(ops.outputUrl ?? ops.inputUrl, ops.output);
|
||||
vi.clearAllMocks();
|
||||
}
|
||||
|
@ -43,6 +51,7 @@ describe("makeSimpleProxyFetcher()", () => {
|
|||
input: {
|
||||
method: "GET",
|
||||
query: {},
|
||||
readHeaders: [],
|
||||
headers: {
|
||||
"X-Hello": "world",
|
||||
},
|
||||
|
@ -62,6 +71,7 @@ describe("makeSimpleProxyFetcher()", () => {
|
|||
input: {
|
||||
method: "GET",
|
||||
headers: {},
|
||||
readHeaders: [],
|
||||
query: {
|
||||
"a": 'b',
|
||||
}
|
||||
|
@ -79,6 +89,7 @@ describe("makeSimpleProxyFetcher()", () => {
|
|||
input: {
|
||||
method: "GET",
|
||||
query: {},
|
||||
readHeaders: [],
|
||||
headers: {},
|
||||
},
|
||||
outputUrl: `https://example.com/proxy?destination=${encodeURIComponent('https://google.com/')}`,
|
||||
|
@ -97,6 +108,7 @@ describe("makeSimpleProxyFetcher()", () => {
|
|||
input: {
|
||||
method: "POST",
|
||||
query: {},
|
||||
readHeaders: [],
|
||||
headers: {},
|
||||
},
|
||||
outputUrl: `https://example.com/proxy?destination=${encodeURIComponent('https://google.com/')}`,
|
||||
|
@ -112,6 +124,7 @@ describe("makeSimpleProxyFetcher()", () => {
|
|||
input: {
|
||||
method: "POST",
|
||||
query: {},
|
||||
readHeaders: [],
|
||||
headers: {},
|
||||
},
|
||||
outputUrl: `https://example.com/proxy?destination=${encodeURIComponent('https://google.com/')}`,
|
||||
|
|
|
@ -16,6 +16,8 @@ describe("makeStandardFetcher()", () => {
|
|||
headers: new Headers({
|
||||
"content-type": "text/plain",
|
||||
}),
|
||||
status: 204,
|
||||
url: "test123",
|
||||
text() {
|
||||
return Promise.resolve(value);
|
||||
},
|
||||
|
@ -24,6 +26,8 @@ describe("makeStandardFetcher()", () => {
|
|||
headers: new Headers({
|
||||
"content-type": "application/json",
|
||||
}),
|
||||
status: 204,
|
||||
url: "test123",
|
||||
json() {
|
||||
return Promise.resolve(value);
|
||||
},
|
||||
|
@ -31,7 +35,11 @@ describe("makeStandardFetcher()", () => {
|
|||
}
|
||||
|
||||
function expectFetchCall(ops: { inputUrl: string, input: DefaultedFetcherOptions, outputUrl?: string, output: any, outputBody: any }) {
|
||||
expect(fetcher(ops.inputUrl, ops.input)).resolves.toEqual(ops.outputBody);
|
||||
const prom = fetcher(ops.inputUrl, ops.input);
|
||||
expect((async () => (await prom).body)()).resolves.toEqual(ops.outputBody);
|
||||
expect((async () => (await prom).headers.entries())()).resolves.toEqual((new Headers()).entries());
|
||||
expect((async () => (await prom).statusCode)()).resolves.toEqual(204);
|
||||
expect((async () => (await prom).finalUrl)()).resolves.toEqual("test123");
|
||||
expect(fetch).toBeCalledWith(ops.outputUrl ?? ops.inputUrl, ops.output);
|
||||
vi.clearAllMocks();
|
||||
}
|
||||
|
@ -43,6 +51,7 @@ describe("makeStandardFetcher()", () => {
|
|||
input: {
|
||||
method: "GET",
|
||||
query: {},
|
||||
readHeaders: [],
|
||||
headers: {
|
||||
"X-Hello": "world",
|
||||
},
|
||||
|
@ -53,6 +62,7 @@ describe("makeStandardFetcher()", () => {
|
|||
headers: {
|
||||
"X-Hello": "world",
|
||||
},
|
||||
body: undefined,
|
||||
},
|
||||
outputBody: "hello world"
|
||||
})
|
||||
|
@ -62,6 +72,7 @@ describe("makeStandardFetcher()", () => {
|
|||
input: {
|
||||
method: "GET",
|
||||
headers: {},
|
||||
readHeaders: [],
|
||||
query: {
|
||||
"a": 'b',
|
||||
}
|
||||
|
@ -79,6 +90,7 @@ describe("makeStandardFetcher()", () => {
|
|||
input: {
|
||||
query: {},
|
||||
headers: {},
|
||||
readHeaders: [],
|
||||
method: "GET"
|
||||
},
|
||||
outputUrl: "https://google.com/",
|
||||
|
@ -97,6 +109,7 @@ describe("makeStandardFetcher()", () => {
|
|||
input: {
|
||||
query: {},
|
||||
headers: {},
|
||||
readHeaders: [],
|
||||
method: "POST"
|
||||
},
|
||||
outputUrl: "https://google.com/",
|
||||
|
@ -112,6 +125,7 @@ describe("makeStandardFetcher()", () => {
|
|||
input: {
|
||||
query: {},
|
||||
headers: {},
|
||||
readHeaders: [],
|
||||
method: "POST"
|
||||
},
|
||||
outputUrl: "https://google.com/",
|
||||
|
|
|
@ -15,40 +15,52 @@ export function makeProviderMocks() {
|
|||
|
||||
const sourceA = {
|
||||
id: 'a',
|
||||
name: 'A',
|
||||
rank: 1,
|
||||
disabled: false,
|
||||
flags: [],
|
||||
} as Sourcerer;
|
||||
const sourceB = {
|
||||
id: 'b',
|
||||
name: 'B',
|
||||
rank: 2,
|
||||
disabled: false,
|
||||
flags: [],
|
||||
} as Sourcerer;
|
||||
const sourceCDisabled = {
|
||||
id: 'c',
|
||||
name: 'C',
|
||||
rank: 3,
|
||||
disabled: true,
|
||||
flags: [],
|
||||
} as Sourcerer;
|
||||
const sourceAHigherRank = {
|
||||
id: 'a',
|
||||
name: 'A',
|
||||
rank: 100,
|
||||
disabled: false,
|
||||
flags: [],
|
||||
} as Sourcerer;
|
||||
const sourceGSameRankAsA = {
|
||||
id: 'g',
|
||||
name: 'G',
|
||||
rank: 1,
|
||||
disabled: false,
|
||||
flags: [],
|
||||
} as Sourcerer;
|
||||
const fullSourceYMovie = {
|
||||
id: 'y',
|
||||
name: 'Y',
|
||||
rank: 105,
|
||||
scrapeMovie: vi.fn(),
|
||||
flags: [],
|
||||
} as Sourcerer;
|
||||
const fullSourceYShow = {
|
||||
id: 'y',
|
||||
name: 'Y',
|
||||
rank: 105,
|
||||
scrapeShow: vi.fn(),
|
||||
flags: [],
|
||||
} as Sourcerer;
|
||||
const fullSourceZBoth = {
|
||||
id: 'z',
|
||||
|
@ -56,6 +68,7 @@ const fullSourceZBoth = {
|
|||
rank: 106,
|
||||
scrapeMovie: vi.fn(),
|
||||
scrapeShow: vi.fn(),
|
||||
flags: [],
|
||||
} as Sourcerer;
|
||||
|
||||
const embedD = {
|
||||
|
|
|
@ -1,12 +1,15 @@
|
|||
import { mockEmbeds, mockSources } from '@/__test__/providerTests';
|
||||
import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers';
|
||||
import { FeatureMap } from '@/entrypoint/utils/targets';
|
||||
import { getProviders } from '@/providers/get';
|
||||
import { vi, describe, it, expect, afterEach } from 'vitest';
|
||||
|
||||
const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks());
|
||||
const mocks = await vi.hoisted(async () => (await import('../providerTests')).makeProviderMocks());
|
||||
vi.mock('@/providers/all', () => mocks);
|
||||
|
||||
const features = {
|
||||
const features: FeatureMap = {
|
||||
requires: [],
|
||||
disallowed: []
|
||||
}
|
||||
|
||||
describe('getProviders()', () => {
|
||||
|
@ -17,7 +20,10 @@ describe('getProviders()', () => {
|
|||
it('should return providers', () => {
|
||||
mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD]);
|
||||
mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]);
|
||||
expect(getProviders(features)).toEqual({
|
||||
expect(getProviders(features, {
|
||||
embeds: getBuiltinEmbeds(),
|
||||
sources: getBuiltinSources(),
|
||||
})).toEqual({
|
||||
sources: [mockSources.sourceA, mockSources.sourceB],
|
||||
embeds: [mockEmbeds.embedD],
|
||||
});
|
||||
|
@ -26,7 +32,10 @@ describe('getProviders()', () => {
|
|||
it('should filter out disabled providers', () => {
|
||||
mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedEDisabled]);
|
||||
mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceCDisabled, mockSources.sourceB]);
|
||||
expect(getProviders(features)).toEqual({
|
||||
expect(getProviders(features,{
|
||||
embeds: getBuiltinEmbeds(),
|
||||
sources: getBuiltinSources(),
|
||||
})).toEqual({
|
||||
sources: [mockSources.sourceA, mockSources.sourceB],
|
||||
embeds: [mockEmbeds.embedD],
|
||||
});
|
||||
|
@ -35,31 +44,46 @@ describe('getProviders()', () => {
|
|||
it('should throw on duplicate ids in sources', () => {
|
||||
mocks.gatherAllEmbeds.mockReturnValue([]);
|
||||
mocks.gatherAllSources.mockReturnValue([mockSources.sourceAHigherRank, mockSources.sourceA, mockSources.sourceB]);
|
||||
expect(() => getProviders(features)).toThrowError();
|
||||
expect(() => getProviders(features,{
|
||||
embeds: getBuiltinEmbeds(),
|
||||
sources: getBuiltinSources(),
|
||||
})).toThrowError();
|
||||
});
|
||||
|
||||
it('should throw on duplicate ids in embeds', () => {
|
||||
mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedDHigherRank, mockEmbeds.embedA]);
|
||||
mocks.gatherAllSources.mockReturnValue([]);
|
||||
expect(() => getProviders(features)).toThrowError();
|
||||
expect(() => getProviders(features,{
|
||||
embeds: getBuiltinEmbeds(),
|
||||
sources: getBuiltinSources(),
|
||||
})).toThrowError();
|
||||
});
|
||||
|
||||
it('should throw on duplicate ids between sources and embeds', () => {
|
||||
mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedA]);
|
||||
mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]);
|
||||
expect(() => getProviders(features)).toThrowError();
|
||||
expect(() => getProviders(features,{
|
||||
embeds: getBuiltinEmbeds(),
|
||||
sources: getBuiltinSources(),
|
||||
})).toThrowError();
|
||||
});
|
||||
|
||||
it('should throw on duplicate rank between sources and embeds', () => {
|
||||
mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedA]);
|
||||
mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]);
|
||||
expect(() => getProviders(features)).toThrowError();
|
||||
expect(() => getProviders(features,{
|
||||
embeds: getBuiltinEmbeds(),
|
||||
sources: getBuiltinSources(),
|
||||
})).toThrowError();
|
||||
});
|
||||
|
||||
it('should not throw with same rank between sources and embeds', () => {
|
||||
mocks.gatherAllEmbeds.mockReturnValue([mockEmbeds.embedD, mockEmbeds.embedHSameRankAsSourceA]);
|
||||
mocks.gatherAllSources.mockReturnValue([mockSources.sourceA, mockSources.sourceB]);
|
||||
expect(getProviders(features)).toEqual({
|
||||
expect(getProviders(features,{
|
||||
embeds: getBuiltinEmbeds(),
|
||||
sources: getBuiltinSources(),
|
||||
})).toEqual({
|
||||
sources: [mockSources.sourceA, mockSources.sourceB],
|
||||
embeds: [mockEmbeds.embedD, mockEmbeds.embedHSameRankAsSourceA],
|
||||
});
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import { mockEmbeds, mockSources } from '@/__test__/providerTests';
|
||||
import { makeProviders } from '@/main/builder';
|
||||
import { targets } from '@/main/targets.ts';
|
||||
import { makeProviders } from '@/entrypoint/declare';
|
||||
import { targets } from '@/entrypoint/utils/targets';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks());
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import { mockEmbeds, mockSources } from '@/__test__/providerTests';
|
||||
import { makeProviders } from '@/main/builder';
|
||||
import { targets } from '@/main/targets.ts';
|
||||
import { makeProviders } from '@/entrypoint/declare';
|
||||
import { targets } from '@/entrypoint/utils/targets';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
const mocks = await vi.hoisted(async () => (await import('../providerTests.ts')).makeProviderMocks());
|
||||
|
|
|
@ -0,0 +1,77 @@
|
|||
import { FeatureMap, Flags, flags, flagsAllowedInFeatures } from "@/entrypoint/utils/targets";
|
||||
import { describe, it, expect } from "vitest";
|
||||
|
||||
describe('flagsAllowedInFeatures()', () => {
|
||||
function checkFeatures(featureMap: FeatureMap, flags: Flags[], output: boolean) {
|
||||
expect(flagsAllowedInFeatures(featureMap, flags)).toEqual(output);
|
||||
}
|
||||
|
||||
it('should check required correctly', () => {
|
||||
checkFeatures({
|
||||
requires: [],
|
||||
disallowed: []
|
||||
}, [], true);
|
||||
checkFeatures({
|
||||
requires: [flags.CORS_ALLOWED],
|
||||
disallowed: []
|
||||
}, [flags.CORS_ALLOWED], true);
|
||||
checkFeatures({
|
||||
requires: [flags.CORS_ALLOWED],
|
||||
disallowed: []
|
||||
}, [], false);
|
||||
checkFeatures({
|
||||
requires: [flags.CORS_ALLOWED, flags.IP_LOCKED],
|
||||
disallowed: []
|
||||
}, [flags.CORS_ALLOWED, flags.IP_LOCKED], true);
|
||||
checkFeatures({
|
||||
requires: [flags.IP_LOCKED],
|
||||
disallowed: []
|
||||
}, [flags.CORS_ALLOWED], false);
|
||||
checkFeatures({
|
||||
requires: [flags.IP_LOCKED],
|
||||
disallowed: []
|
||||
}, [], false);
|
||||
});
|
||||
|
||||
it('should check disallowed correctly', () => {
|
||||
checkFeatures({
|
||||
requires: [],
|
||||
disallowed: []
|
||||
}, [], true);
|
||||
checkFeatures({
|
||||
requires: [],
|
||||
disallowed: [flags.CORS_ALLOWED]
|
||||
}, [], true);
|
||||
checkFeatures({
|
||||
requires: [],
|
||||
disallowed: [flags.CORS_ALLOWED]
|
||||
}, [flags.CORS_ALLOWED], false);
|
||||
checkFeatures({
|
||||
requires: [],
|
||||
disallowed: [flags.CORS_ALLOWED]
|
||||
}, [flags.IP_LOCKED], true);
|
||||
checkFeatures({
|
||||
requires: [],
|
||||
disallowed: [flags.CORS_ALLOWED, flags.IP_LOCKED]
|
||||
}, [flags.CORS_ALLOWED], false);
|
||||
});
|
||||
|
||||
it('should pass mixed tests', () => {
|
||||
checkFeatures({
|
||||
requires: [flags.CORS_ALLOWED],
|
||||
disallowed: [flags.IP_LOCKED]
|
||||
}, [], false);
|
||||
checkFeatures({
|
||||
requires: [flags.CORS_ALLOWED],
|
||||
disallowed: [flags.IP_LOCKED]
|
||||
}, [flags.CORS_ALLOWED], true);
|
||||
checkFeatures({
|
||||
requires: [flags.CORS_ALLOWED],
|
||||
disallowed: [flags.IP_LOCKED]
|
||||
}, [flags.IP_LOCKED], false);
|
||||
checkFeatures({
|
||||
requires: [flags.CORS_ALLOWED],
|
||||
disallowed: [flags.IP_LOCKED]
|
||||
}, [flags.IP_LOCKED, flags.CORS_ALLOWED], false);
|
||||
});
|
||||
});
|
|
@ -9,7 +9,9 @@ describe('isValidStream()', () => {
|
|||
it('should pass valid streams', () => {
|
||||
expect(isValidStream({
|
||||
type: "file",
|
||||
id: "a",
|
||||
flags: [],
|
||||
captions: [],
|
||||
qualities: {
|
||||
"1080": {
|
||||
type: "mp4",
|
||||
|
@ -19,7 +21,9 @@ describe('isValidStream()', () => {
|
|||
})).toBe(true);
|
||||
expect(isValidStream({
|
||||
type: "hls",
|
||||
id: "a",
|
||||
flags: [],
|
||||
captions: [],
|
||||
playlist: "hello-world"
|
||||
})).toBe(true);
|
||||
});
|
||||
|
@ -27,7 +31,9 @@ describe('isValidStream()', () => {
|
|||
it('should detect empty qualities', () => {
|
||||
expect(isValidStream({
|
||||
type: "file",
|
||||
id: "a",
|
||||
flags: [],
|
||||
captions: [],
|
||||
qualities: {}
|
||||
})).toBe(false);
|
||||
});
|
||||
|
@ -35,7 +41,9 @@ describe('isValidStream()', () => {
|
|||
it('should detect empty stream urls', () => {
|
||||
expect(isValidStream({
|
||||
type: "file",
|
||||
id: "a",
|
||||
flags: [],
|
||||
captions: [],
|
||||
qualities: {
|
||||
"1080": {
|
||||
type: "mp4",
|
||||
|
@ -48,7 +56,9 @@ describe('isValidStream()', () => {
|
|||
it('should detect emtpy HLS playlists', () => {
|
||||
expect(isValidStream({
|
||||
type: "hls",
|
||||
id: "a",
|
||||
flags: [],
|
||||
captions: [],
|
||||
playlist: "",
|
||||
})).toBe(false);
|
||||
});
|
||||
|
|
430
src/dev-cli.ts
430
src/dev-cli.ts
|
@ -1,430 +0,0 @@
|
|||
/* eslint import/no-extraneous-dependencies: ["error", {"devDependencies": true}] */
|
||||
|
||||
import util from 'node:util';
|
||||
|
||||
import { program } from 'commander';
|
||||
import dotenv from 'dotenv';
|
||||
import { prompt } from 'enquirer';
|
||||
import nodeFetch from 'node-fetch';
|
||||
import Spinnies from 'spinnies';
|
||||
|
||||
import { MetaOutput, MovieMedia, ProviderControls, ShowMedia, makeProviders, makeStandardFetcher, targets } from '.';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
type ProviderSourceAnswers = {
|
||||
id: string;
|
||||
type: string;
|
||||
};
|
||||
|
||||
type EmbedSourceAnswers = {
|
||||
url: string;
|
||||
};
|
||||
|
||||
type CommonAnswers = {
|
||||
fetcher: string;
|
||||
source: string;
|
||||
};
|
||||
|
||||
type ShowAnswers = {
|
||||
season: string;
|
||||
episode: string;
|
||||
};
|
||||
|
||||
type CommandLineArguments = {
|
||||
fetcher: string;
|
||||
sourceId: string;
|
||||
tmdbId: string;
|
||||
type: string;
|
||||
season: string;
|
||||
episode: string;
|
||||
url: string;
|
||||
headers?: Record<string, string>;
|
||||
};
|
||||
|
||||
const TMDB_API_KEY = process.env.MOVIE_WEB_TMDB_API_KEY ?? '';
|
||||
|
||||
if (!TMDB_API_KEY?.trim()) {
|
||||
throw new Error('Missing MOVIE_WEB_TMDB_API_KEY environment variable');
|
||||
}
|
||||
|
||||
function logDeepObject(object: Record<any, any>) {
|
||||
console.log(util.inspect(object, { showHidden: false, depth: null, colors: true }));
|
||||
}
|
||||
|
||||
function getAllSources() {
|
||||
// * The only way to get a list of all sources is to
|
||||
// * create all these things. Maybe this should change
|
||||
const providers = makeProviders({
|
||||
fetcher: makeStandardFetcher(nodeFetch),
|
||||
target: targets.NATIVE,
|
||||
});
|
||||
|
||||
const combined = [...providers.listSources(), ...providers.listEmbeds()];
|
||||
|
||||
// * Remove dupes
|
||||
const map = new Map(combined.map((source) => [source.id, source]));
|
||||
|
||||
return [...map.values()];
|
||||
}
|
||||
|
||||
// * Defined here cuz ESLint didn't like the order these were defined in
|
||||
const sources = getAllSources();
|
||||
|
||||
async function makeTMDBRequest(url: string): Promise<Response> {
|
||||
const headers: {
|
||||
accept: 'application/json';
|
||||
authorization?: string;
|
||||
} = {
|
||||
accept: 'application/json',
|
||||
};
|
||||
|
||||
// * Used to get around ESLint
|
||||
// * Assignment to function parameter 'url'. eslint (no-param-reassign)
|
||||
let requestURL = url;
|
||||
|
||||
// * JWT keys always start with ey and are ONLY valid as a header.
|
||||
// * All other keys are ONLY valid as a query param.
|
||||
// * Thanks TMDB.
|
||||
if (TMDB_API_KEY.startsWith('ey')) {
|
||||
headers.authorization = `Bearer ${TMDB_API_KEY}`;
|
||||
} else {
|
||||
requestURL += `?api_key=${TMDB_API_KEY}`;
|
||||
}
|
||||
|
||||
return fetch(requestURL, {
|
||||
method: 'GET',
|
||||
headers,
|
||||
});
|
||||
}
|
||||
|
||||
async function getMovieMediaDetails(id: string): Promise<MovieMedia> {
|
||||
const response = await makeTMDBRequest(`https://api.themoviedb.org/3/movie/${id}`);
|
||||
const movie = await response.json();
|
||||
|
||||
if (movie.success === false) {
|
||||
throw new Error(movie.status_message);
|
||||
}
|
||||
|
||||
if (!movie.release_date) {
|
||||
throw new Error(`${movie.title} has no release_date. Assuming unreleased`);
|
||||
}
|
||||
|
||||
return {
|
||||
type: 'movie',
|
||||
title: movie.title,
|
||||
releaseYear: Number(movie.release_date.split('-')[0]),
|
||||
tmdbId: id,
|
||||
};
|
||||
}
|
||||
|
||||
async function getShowMediaDetails(id: string, seasonNumber: string, episodeNumber: string): Promise<ShowMedia> {
|
||||
// * TV shows require the TMDB ID for the series, season, and episode
|
||||
// * and the name of the series. Needs multiple requests
|
||||
let response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}`);
|
||||
const series = await response.json();
|
||||
|
||||
if (series.success === false) {
|
||||
throw new Error(series.status_message);
|
||||
}
|
||||
|
||||
if (!series.first_air_date) {
|
||||
throw new Error(`${series.name} has no first_air_date. Assuming unaired`);
|
||||
}
|
||||
|
||||
response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}`);
|
||||
const season = await response.json();
|
||||
|
||||
if (season.success === false) {
|
||||
throw new Error(season.status_message);
|
||||
}
|
||||
|
||||
response = await makeTMDBRequest(
|
||||
`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}/episode/${episodeNumber}`,
|
||||
);
|
||||
const episode = await response.json();
|
||||
|
||||
if (episode.success === false) {
|
||||
throw new Error(episode.status_message);
|
||||
}
|
||||
|
||||
return {
|
||||
type: 'show',
|
||||
title: series.name,
|
||||
releaseYear: Number(series.first_air_date.split('-')[0]),
|
||||
tmdbId: id,
|
||||
episode: {
|
||||
number: episode.episode_number,
|
||||
tmdbId: episode.id,
|
||||
},
|
||||
season: {
|
||||
number: season.season_number,
|
||||
tmdbId: season.id,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function joinMediaTypes(mediaTypes: string[] | undefined) {
|
||||
if (mediaTypes) {
|
||||
const formatted = mediaTypes
|
||||
.map((type: string) => {
|
||||
return `${type[0].toUpperCase() + type.substring(1).toLowerCase()}s`;
|
||||
})
|
||||
.join(' / ');
|
||||
|
||||
return `(${formatted})`;
|
||||
}
|
||||
return ''; // * Embed sources pass through here too
|
||||
}
|
||||
|
||||
async function runScraper(providers: ProviderControls, source: MetaOutput, options: CommandLineArguments) {
|
||||
const spinnies = new Spinnies();
|
||||
|
||||
if (source.type === 'embed') {
|
||||
spinnies.add('scrape', { text: `Running ${source.name} scraper on ${options.url}` });
|
||||
try {
|
||||
const result = await providers.runEmbedScraper({
|
||||
url: options.url,
|
||||
id: source.id,
|
||||
headers: options.headers,
|
||||
});
|
||||
spinnies.succeed('scrape', { text: 'Done!' });
|
||||
logDeepObject(result);
|
||||
} catch (error) {
|
||||
let message = 'Unknown error';
|
||||
if (error instanceof Error) {
|
||||
message = error.message;
|
||||
}
|
||||
|
||||
spinnies.fail('scrape', { text: `ERROR: ${message}` });
|
||||
}
|
||||
} else {
|
||||
let media;
|
||||
|
||||
if (options.type === 'movie') {
|
||||
media = await getMovieMediaDetails(options.tmdbId);
|
||||
} else {
|
||||
media = await getShowMediaDetails(options.tmdbId, options.season, options.episode);
|
||||
}
|
||||
|
||||
spinnies.add('scrape', { text: `Running ${source.name} scraper on ${media.title}` });
|
||||
try {
|
||||
const result = await providers.runSourceScraper({
|
||||
media,
|
||||
id: source.id,
|
||||
});
|
||||
spinnies.succeed('scrape', { text: 'Done!' });
|
||||
logDeepObject(result);
|
||||
} catch (error) {
|
||||
let message = 'Unknown error';
|
||||
if (error instanceof Error) {
|
||||
message = error.message;
|
||||
}
|
||||
|
||||
spinnies.fail('scrape', { text: `ERROR: ${message}` });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function processOptions(options: CommandLineArguments) {
|
||||
if (options.fetcher !== 'node-fetch' && options.fetcher !== 'native') {
|
||||
throw new Error("Fetcher must be either 'native' or 'node-fetch'");
|
||||
}
|
||||
|
||||
if (!options.sourceId.trim()) {
|
||||
throw new Error('Source ID must be provided');
|
||||
}
|
||||
|
||||
const source = sources.find(({ id }) => id === options.sourceId);
|
||||
|
||||
if (!source) {
|
||||
throw new Error('Invalid source ID. No source found');
|
||||
}
|
||||
|
||||
if (source.type === 'embed' && !options.url.trim()) {
|
||||
throw new Error('Must provide an embed URL for embed sources');
|
||||
}
|
||||
|
||||
if (source.type === 'source') {
|
||||
if (!options.tmdbId.trim()) {
|
||||
throw new Error('Must provide a TMDB ID for provider sources');
|
||||
}
|
||||
|
||||
if (Number.isNaN(Number(options.tmdbId)) || Number(options.tmdbId) < 0) {
|
||||
throw new Error('TMDB ID must be a number greater than 0');
|
||||
}
|
||||
|
||||
if (!options.type.trim()) {
|
||||
throw new Error('Must provide a type for provider sources');
|
||||
}
|
||||
|
||||
if (options.type !== 'movie' && options.type !== 'show') {
|
||||
throw new Error("Invalid media type. Must be either 'movie' or 'show'");
|
||||
}
|
||||
|
||||
if (options.type === 'show') {
|
||||
if (!options.season.trim()) {
|
||||
throw new Error('Must provide a season number for TV shows');
|
||||
}
|
||||
|
||||
if (!options.episode.trim()) {
|
||||
throw new Error('Must provide an episode number for TV shows');
|
||||
}
|
||||
|
||||
if (Number.isNaN(Number(options.season)) || Number(options.season) <= 0) {
|
||||
throw new Error('Season number must be a number greater than 0');
|
||||
}
|
||||
|
||||
if (Number.isNaN(Number(options.episode)) || Number(options.episode) <= 0) {
|
||||
throw new Error('Episode number must be a number greater than 0');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof options.headers === 'string') {
|
||||
options.headers = JSON.parse(options.headers);
|
||||
}
|
||||
|
||||
let fetcher;
|
||||
|
||||
if (options.fetcher === 'native') {
|
||||
fetcher = makeStandardFetcher(fetch);
|
||||
} else {
|
||||
fetcher = makeStandardFetcher(nodeFetch);
|
||||
}
|
||||
|
||||
const providers = makeProviders({
|
||||
fetcher,
|
||||
target: targets.NATIVE,
|
||||
});
|
||||
|
||||
await runScraper(providers, source, options);
|
||||
}
|
||||
|
||||
async function runQuestions() {
|
||||
const options = {
|
||||
fetcher: 'node-fetch',
|
||||
sourceId: '',
|
||||
tmdbId: '',
|
||||
type: 'movie',
|
||||
season: '0',
|
||||
episode: '0',
|
||||
url: '',
|
||||
};
|
||||
|
||||
const answers = await prompt<CommonAnswers>([
|
||||
{
|
||||
type: 'select',
|
||||
name: 'fetcher',
|
||||
message: 'Select a fetcher',
|
||||
choices: [
|
||||
{
|
||||
message: 'Native',
|
||||
name: 'native',
|
||||
},
|
||||
{
|
||||
message: 'Node fetch',
|
||||
name: 'node-fetch',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'select',
|
||||
name: 'source',
|
||||
message: 'Select a source',
|
||||
choices: sources.map((source) => ({
|
||||
message: `[${source.type.toLocaleUpperCase()}] ${source.name} ${joinMediaTypes(source.mediaTypes)}`.trim(),
|
||||
name: source.id,
|
||||
})),
|
||||
},
|
||||
]);
|
||||
|
||||
options.fetcher = answers.fetcher;
|
||||
options.sourceId = answers.source;
|
||||
|
||||
const source = sources.find(({ id }) => id === answers.source);
|
||||
|
||||
if (!source) {
|
||||
throw new Error(`No source with ID ${answers.source} found`);
|
||||
}
|
||||
|
||||
if (source.type === 'embed') {
|
||||
const sourceAnswers = await prompt<EmbedSourceAnswers>([
|
||||
{
|
||||
type: 'input',
|
||||
name: 'url',
|
||||
message: 'Embed URL',
|
||||
},
|
||||
]);
|
||||
|
||||
options.url = sourceAnswers.url;
|
||||
} else {
|
||||
const sourceAnswers = await prompt<ProviderSourceAnswers>([
|
||||
{
|
||||
type: 'input',
|
||||
name: 'id',
|
||||
message: 'TMDB ID',
|
||||
},
|
||||
{
|
||||
type: 'select',
|
||||
name: 'type',
|
||||
message: 'Media type',
|
||||
choices: [
|
||||
{
|
||||
message: 'Movie',
|
||||
name: 'movie',
|
||||
},
|
||||
{
|
||||
message: 'TV Show',
|
||||
name: 'show',
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
|
||||
options.tmdbId = sourceAnswers.id;
|
||||
options.type = sourceAnswers.type;
|
||||
|
||||
if (sourceAnswers.type === 'show') {
|
||||
const seriesAnswers = await prompt<ShowAnswers>([
|
||||
{
|
||||
type: 'input',
|
||||
name: 'season',
|
||||
message: 'Season',
|
||||
},
|
||||
{
|
||||
type: 'input',
|
||||
name: 'episode',
|
||||
message: 'Episode',
|
||||
},
|
||||
]);
|
||||
|
||||
options.season = seriesAnswers.season;
|
||||
options.episode = seriesAnswers.episode;
|
||||
}
|
||||
}
|
||||
|
||||
await processOptions(options);
|
||||
}
|
||||
|
||||
async function runCommandLine() {
|
||||
program
|
||||
.option('-f, --fetcher <fetcher>', "Fetcher to use. Either 'native' or 'node-fetch'", 'node-fetch')
|
||||
.option('-sid, --source-id <id>', 'ID for the source to use. Either an embed or provider', '')
|
||||
.option('-tid, --tmdb-id <id>', 'TMDB ID for the media to scrape. Only used if source is a provider', '')
|
||||
.option('-t, --type <type>', "Media type. Either 'movie' or 'show'. Only used if source is a provider", 'movie')
|
||||
.option('-s, --season <number>', "Season number. Only used if type is 'show'", '0')
|
||||
.option('-e, --episode <number>', "Episode number. Only used if type is 'show'", '0')
|
||||
.option('-u, --url <embed URL>', 'URL to a video embed. Only used if source is an embed', '')
|
||||
.option('-h, --headers <JSON>', 'Optional headers to pass to scrapers. JSON encoded');
|
||||
|
||||
program.parse();
|
||||
|
||||
await processOptions(program.opts());
|
||||
}
|
||||
|
||||
if (process.argv.length === 2) {
|
||||
runQuestions();
|
||||
} else {
|
||||
runCommandLine();
|
||||
}
|
|
@ -0,0 +1 @@
|
|||
dist
|
|
@ -0,0 +1,11 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Scraper CLI</title>
|
||||
</head>
|
||||
<body>
|
||||
<script src="./index.ts" type="module"></script>
|
||||
</body>
|
||||
</html>
|
|
@ -0,0 +1,17 @@
|
|||
import { makeProviders, makeSimpleProxyFetcher, makeStandardFetcher, targets } from '../../../lib';
|
||||
|
||||
(window as any).scrape = (proxyUrl: string, type: 'source' | 'embed', input: any) => {
|
||||
const providers = makeProviders({
|
||||
fetcher: makeStandardFetcher(fetch),
|
||||
target: targets.BROWSER,
|
||||
proxiedFetcher: makeSimpleProxyFetcher(proxyUrl, fetch),
|
||||
});
|
||||
if (type === 'source') {
|
||||
return providers.runSourceScraper(input);
|
||||
}
|
||||
if (type === 'embed') {
|
||||
return providers.runEmbedScraper(input);
|
||||
}
|
||||
|
||||
throw new Error('Input input type');
|
||||
};
|
|
@ -0,0 +1,16 @@
|
|||
export function getConfig() {
|
||||
let tmdbApiKey = process.env.MOVIE_WEB_TMDB_API_KEY ?? '';
|
||||
tmdbApiKey = tmdbApiKey.trim();
|
||||
|
||||
if (!tmdbApiKey) {
|
||||
throw new Error('Missing MOVIE_WEB_TMDB_API_KEY environment variable');
|
||||
}
|
||||
|
||||
let proxyUrl: undefined | string = process.env.MOVIE_WEB_PROXY_URL;
|
||||
proxyUrl = !proxyUrl ? undefined : proxyUrl;
|
||||
|
||||
return {
|
||||
tmdbApiKey,
|
||||
proxyUrl,
|
||||
};
|
||||
}
|
|
@ -0,0 +1,185 @@
|
|||
/* eslint import/no-extraneous-dependencies: ["error", {"devDependencies": true}] */
|
||||
|
||||
import { program } from 'commander';
|
||||
import dotenv from 'dotenv';
|
||||
import { prompt } from 'enquirer';
|
||||
|
||||
import { runScraper } from '@/dev-cli/scraper';
|
||||
import { processOptions } from '@/dev-cli/validate';
|
||||
|
||||
import { getBuiltinEmbeds, getBuiltinSources } from '..';
|
||||
|
||||
dotenv.config();
|
||||
|
||||
type ProviderSourceAnswers = {
|
||||
id: string;
|
||||
type: string;
|
||||
};
|
||||
|
||||
type EmbedSourceAnswers = {
|
||||
url: string;
|
||||
};
|
||||
|
||||
type CommonAnswers = {
|
||||
fetcher: string;
|
||||
source: string;
|
||||
};
|
||||
|
||||
type ShowAnswers = {
|
||||
season: string;
|
||||
episode: string;
|
||||
};
|
||||
|
||||
const sourceScrapers = getBuiltinSources().sort((a, b) => b.rank - a.rank);
|
||||
const embedScrapers = getBuiltinEmbeds().sort((a, b) => b.rank - a.rank);
|
||||
const sources = [...sourceScrapers, ...embedScrapers];
|
||||
|
||||
function joinMediaTypes(mediaTypes: string[] | undefined) {
|
||||
if (mediaTypes) {
|
||||
const formatted = mediaTypes
|
||||
.map((type: string) => {
|
||||
return `${type[0].toUpperCase() + type.substring(1).toLowerCase()}s`;
|
||||
})
|
||||
.join(' / ');
|
||||
|
||||
return `(${formatted})`;
|
||||
}
|
||||
return ''; // * Embed sources pass through here too
|
||||
}
|
||||
|
||||
async function runQuestions() {
|
||||
const options = {
|
||||
fetcher: 'node-fetch',
|
||||
sourceId: '',
|
||||
tmdbId: '',
|
||||
type: 'movie',
|
||||
season: '0',
|
||||
episode: '0',
|
||||
url: '',
|
||||
};
|
||||
|
||||
const answers = await prompt<CommonAnswers>([
|
||||
{
|
||||
type: 'select',
|
||||
name: 'fetcher',
|
||||
message: 'Select a fetcher mode',
|
||||
choices: [
|
||||
{
|
||||
message: 'Native',
|
||||
name: 'native',
|
||||
},
|
||||
{
|
||||
message: 'Node fetch',
|
||||
name: 'node-fetch',
|
||||
},
|
||||
{
|
||||
message: 'Browser',
|
||||
name: 'browser',
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'select',
|
||||
name: 'source',
|
||||
message: 'Select a source',
|
||||
choices: sources.map((source) => ({
|
||||
message: `[${source.type.toLocaleUpperCase()}] ${source.name} ${joinMediaTypes(source.mediaTypes)}`.trim(),
|
||||
name: source.id,
|
||||
})),
|
||||
},
|
||||
]);
|
||||
|
||||
options.fetcher = answers.fetcher;
|
||||
options.sourceId = answers.source;
|
||||
|
||||
const source = sources.find(({ id }) => id === answers.source);
|
||||
|
||||
if (!source) {
|
||||
throw new Error(`No source with ID ${answers.source} found`);
|
||||
}
|
||||
|
||||
if (source.type === 'embed') {
|
||||
const sourceAnswers = await prompt<EmbedSourceAnswers>([
|
||||
{
|
||||
type: 'input',
|
||||
name: 'url',
|
||||
message: 'Embed URL',
|
||||
},
|
||||
]);
|
||||
|
||||
options.url = sourceAnswers.url;
|
||||
} else {
|
||||
const sourceAnswers = await prompt<ProviderSourceAnswers>([
|
||||
{
|
||||
type: 'input',
|
||||
name: 'id',
|
||||
message: 'TMDB ID',
|
||||
},
|
||||
{
|
||||
type: 'select',
|
||||
name: 'type',
|
||||
message: 'Media type',
|
||||
choices: [
|
||||
{
|
||||
message: 'Movie',
|
||||
name: 'movie',
|
||||
},
|
||||
{
|
||||
message: 'TV Show',
|
||||
name: 'show',
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
|
||||
options.tmdbId = sourceAnswers.id;
|
||||
options.type = sourceAnswers.type;
|
||||
|
||||
if (sourceAnswers.type === 'show') {
|
||||
const seriesAnswers = await prompt<ShowAnswers>([
|
||||
{
|
||||
type: 'input',
|
||||
name: 'season',
|
||||
message: 'Season',
|
||||
},
|
||||
{
|
||||
type: 'input',
|
||||
name: 'episode',
|
||||
message: 'Episode',
|
||||
},
|
||||
]);
|
||||
|
||||
options.season = seriesAnswers.season;
|
||||
options.episode = seriesAnswers.episode;
|
||||
}
|
||||
}
|
||||
|
||||
const { providerOptions, source: validatedSource, options: validatedOps } = await processOptions(sources, options);
|
||||
await runScraper(providerOptions, validatedSource, validatedOps);
|
||||
}
|
||||
|
||||
async function runCommandLine() {
|
||||
program
|
||||
.option('-f, --fetcher <fetcher>', "Fetcher to use. Either 'native' or 'node-fetch'", 'node-fetch')
|
||||
.option('-sid, --source-id <id>', 'ID for the source to use. Either an embed or provider', '')
|
||||
.option('-tid, --tmdb-id <id>', 'TMDB ID for the media to scrape. Only used if source is a provider', '')
|
||||
.option('-t, --type <type>', "Media type. Either 'movie' or 'show'. Only used if source is a provider", 'movie')
|
||||
.option('-s, --season <number>', "Season number. Only used if type is 'show'", '0')
|
||||
.option('-e, --episode <number>', "Episode number. Only used if type is 'show'", '0')
|
||||
.option('-u, --url <embed URL>', 'URL to a video embed. Only used if source is an embed', '');
|
||||
|
||||
program.parse();
|
||||
|
||||
const {
|
||||
providerOptions,
|
||||
source: validatedSource,
|
||||
options: validatedOps,
|
||||
} = await processOptions(sources, program.opts());
|
||||
await runScraper(providerOptions, validatedSource, validatedOps);
|
||||
}
|
||||
|
||||
if (process.argv.length === 2) {
|
||||
runQuestions().catch(() => console.error('Exited.'));
|
||||
} else {
|
||||
runCommandLine().catch(() => console.error('Exited.'));
|
||||
}
|
|
@ -0,0 +1,5 @@
|
|||
import { inspect } from 'node:util';
|
||||
|
||||
export function logDeepObject(object: Record<any, any>) {
|
||||
console.log(inspect(object, { showHidden: false, depth: null, colors: true }));
|
||||
}
|
|
@ -0,0 +1,136 @@
|
|||
/* eslint import/no-extraneous-dependencies: ["error", {"devDependencies": true}] */
|
||||
|
||||
import { existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
import puppeteer, { Browser } from 'puppeteer';
|
||||
import Spinnies from 'spinnies';
|
||||
import { PreviewServer, build, preview } from 'vite';
|
||||
|
||||
import { getConfig } from '@/dev-cli/config';
|
||||
import { logDeepObject } from '@/dev-cli/logging';
|
||||
import { getMovieMediaDetails, getShowMediaDetails } from '@/dev-cli/tmdb';
|
||||
import { CommandLineArguments } from '@/dev-cli/validate';
|
||||
|
||||
import { MetaOutput, ProviderMakerOptions, makeProviders } from '..';
|
||||
|
||||
async function runBrowserScraping(
|
||||
providerOptions: ProviderMakerOptions,
|
||||
source: MetaOutput,
|
||||
options: CommandLineArguments,
|
||||
) {
|
||||
if (!existsSync(join(__dirname, '../../lib/index.mjs')))
|
||||
throw new Error('Please compile before running cli in browser mode');
|
||||
const config = getConfig();
|
||||
if (!config.proxyUrl)
|
||||
throw new Error('Simple proxy url must be set in the environment (MOVIE_WEB_PROXY_URL) for browser mode to work');
|
||||
|
||||
const root = join(__dirname, 'browser');
|
||||
let server: PreviewServer | undefined;
|
||||
let browser: Browser | undefined;
|
||||
try {
|
||||
// setup browser
|
||||
await build({
|
||||
root,
|
||||
});
|
||||
server = await preview({
|
||||
root,
|
||||
});
|
||||
browser = await puppeteer.launch({
|
||||
headless: 'new',
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
const page = await browser.newPage();
|
||||
await page.goto(server.resolvedUrls.local[0]);
|
||||
await page.waitForFunction('!!window.scrape', { timeout: 5000 });
|
||||
|
||||
// get input media
|
||||
let input: any;
|
||||
if (source.type === 'embed') {
|
||||
input = {
|
||||
url: options.url,
|
||||
id: source.id,
|
||||
};
|
||||
} else if (source.type === 'source') {
|
||||
let media;
|
||||
if (options.type === 'movie') {
|
||||
media = await getMovieMediaDetails(options.tmdbId);
|
||||
} else {
|
||||
media = await getShowMediaDetails(options.tmdbId, options.season, options.episode);
|
||||
}
|
||||
input = {
|
||||
media,
|
||||
id: source.id,
|
||||
};
|
||||
} else {
|
||||
throw new Error('Wrong source input type');
|
||||
}
|
||||
|
||||
return await page.evaluate(
|
||||
async (proxy, type, inp) => {
|
||||
return (window as any).scrape(proxy, type, inp);
|
||||
},
|
||||
config.proxyUrl,
|
||||
source.type,
|
||||
input,
|
||||
);
|
||||
} finally {
|
||||
server?.httpServer.close();
|
||||
await browser?.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function runActualScraping(
|
||||
providerOptions: ProviderMakerOptions,
|
||||
source: MetaOutput,
|
||||
options: CommandLineArguments,
|
||||
): Promise<any> {
|
||||
if (options.fetcher === 'browser') return runBrowserScraping(providerOptions, source, options);
|
||||
const providers = makeProviders(providerOptions);
|
||||
|
||||
if (source.type === 'embed') {
|
||||
return providers.runEmbedScraper({
|
||||
url: options.url,
|
||||
id: source.id,
|
||||
});
|
||||
}
|
||||
|
||||
if (source.type === 'source') {
|
||||
let media;
|
||||
|
||||
if (options.type === 'movie') {
|
||||
media = await getMovieMediaDetails(options.tmdbId);
|
||||
} else {
|
||||
media = await getShowMediaDetails(options.tmdbId, options.season, options.episode);
|
||||
}
|
||||
|
||||
return providers.runSourceScraper({
|
||||
media,
|
||||
id: source.id,
|
||||
});
|
||||
}
|
||||
|
||||
throw new Error('Invalid source type');
|
||||
}
|
||||
|
||||
export async function runScraper(
|
||||
providerOptions: ProviderMakerOptions,
|
||||
source: MetaOutput,
|
||||
options: CommandLineArguments,
|
||||
) {
|
||||
const spinnies = new Spinnies();
|
||||
|
||||
spinnies.add('scrape', { text: `Running ${source.name} scraper` });
|
||||
try {
|
||||
const result = await runActualScraping(providerOptions, source, options);
|
||||
spinnies.succeed('scrape', { text: 'Done!' });
|
||||
logDeepObject(result);
|
||||
} catch (error) {
|
||||
let message = 'Unknown error';
|
||||
if (error instanceof Error) {
|
||||
message = error.message;
|
||||
}
|
||||
spinnies.fail('scrape', { text: `ERROR: ${message}` });
|
||||
console.error(error);
|
||||
}
|
||||
}
|
|
@ -0,0 +1,95 @@
|
|||
import { getConfig } from '@/dev-cli/config';
|
||||
|
||||
import { MovieMedia, ShowMedia } from '..';
|
||||
|
||||
export async function makeTMDBRequest(url: string): Promise<Response> {
|
||||
const headers: {
|
||||
accept: 'application/json';
|
||||
authorization?: string;
|
||||
} = {
|
||||
accept: 'application/json',
|
||||
};
|
||||
|
||||
let requestURL = url;
|
||||
const key = getConfig().tmdbApiKey;
|
||||
|
||||
// * JWT keys always start with ey and are ONLY valid as a header.
|
||||
// * All other keys are ONLY valid as a query param.
|
||||
// * Thanks TMDB.
|
||||
if (key.startsWith('ey')) {
|
||||
headers.authorization = `Bearer ${key}`;
|
||||
} else {
|
||||
requestURL += `?api_key=${key}`;
|
||||
}
|
||||
|
||||
return fetch(requestURL, {
|
||||
method: 'GET',
|
||||
headers,
|
||||
});
|
||||
}
|
||||
|
||||
export async function getMovieMediaDetails(id: string): Promise<MovieMedia> {
|
||||
const response = await makeTMDBRequest(`https://api.themoviedb.org/3/movie/${id}`);
|
||||
const movie = await response.json();
|
||||
|
||||
if (movie.success === false) {
|
||||
throw new Error(movie.status_message);
|
||||
}
|
||||
|
||||
if (!movie.release_date) {
|
||||
throw new Error(`${movie.title} has no release_date. Assuming unreleased`);
|
||||
}
|
||||
|
||||
return {
|
||||
type: 'movie',
|
||||
title: movie.title,
|
||||
releaseYear: Number(movie.release_date.split('-')[0]),
|
||||
tmdbId: id,
|
||||
};
|
||||
}
|
||||
|
||||
export async function getShowMediaDetails(id: string, seasonNumber: string, episodeNumber: string): Promise<ShowMedia> {
|
||||
// * TV shows require the TMDB ID for the series, season, and episode
|
||||
// * and the name of the series. Needs multiple requests
|
||||
let response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}`);
|
||||
const series = await response.json();
|
||||
|
||||
if (series.success === false) {
|
||||
throw new Error(series.status_message);
|
||||
}
|
||||
|
||||
if (!series.first_air_date) {
|
||||
throw new Error(`${series.name} has no first_air_date. Assuming unaired`);
|
||||
}
|
||||
|
||||
response = await makeTMDBRequest(`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}`);
|
||||
const season = await response.json();
|
||||
|
||||
if (season.success === false) {
|
||||
throw new Error(season.status_message);
|
||||
}
|
||||
|
||||
response = await makeTMDBRequest(
|
||||
`https://api.themoviedb.org/3/tv/${id}/season/${seasonNumber}/episode/${episodeNumber}`,
|
||||
);
|
||||
const episode = await response.json();
|
||||
|
||||
if (episode.success === false) {
|
||||
throw new Error(episode.status_message);
|
||||
}
|
||||
|
||||
return {
|
||||
type: 'show',
|
||||
title: series.name,
|
||||
releaseYear: Number(series.first_air_date.split('-')[0]),
|
||||
tmdbId: id,
|
||||
episode: {
|
||||
number: episode.episode_number,
|
||||
tmdbId: episode.id,
|
||||
},
|
||||
season: {
|
||||
number: season.season_number,
|
||||
tmdbId: season.id,
|
||||
},
|
||||
};
|
||||
}
|
|
@ -0,0 +1,91 @@
|
|||
import nodeFetch from 'node-fetch';
|
||||
|
||||
import { Embed, Sourcerer } from '@/providers/base';
|
||||
|
||||
import { ProviderMakerOptions, makeStandardFetcher, targets } from '..';
|
||||
|
||||
export type CommandLineArguments = {
|
||||
fetcher: string;
|
||||
sourceId: string;
|
||||
tmdbId: string;
|
||||
type: string;
|
||||
season: string;
|
||||
episode: string;
|
||||
url: string;
|
||||
};
|
||||
|
||||
export async function processOptions(sources: Array<Embed | Sourcerer>, options: CommandLineArguments) {
|
||||
const fetcherOptions = ['node-fetch', 'native', 'browser'];
|
||||
if (!fetcherOptions.includes(options.fetcher)) {
|
||||
throw new Error(`Fetcher must be any of: ${fetcherOptions.join()}`);
|
||||
}
|
||||
|
||||
if (!options.sourceId.trim()) {
|
||||
throw new Error('Source ID must be provided');
|
||||
}
|
||||
|
||||
const source = sources.find(({ id }) => id === options.sourceId);
|
||||
|
||||
if (!source) {
|
||||
throw new Error('Invalid source ID. No source found');
|
||||
}
|
||||
|
||||
if (source.type === 'embed' && !options.url.trim()) {
|
||||
throw new Error('Must provide an embed URL for embed sources');
|
||||
}
|
||||
|
||||
if (source.type === 'source') {
|
||||
if (!options.tmdbId.trim()) {
|
||||
throw new Error('Must provide a TMDB ID for provider sources');
|
||||
}
|
||||
|
||||
if (Number.isNaN(Number(options.tmdbId)) || Number(options.tmdbId) < 0) {
|
||||
throw new Error('TMDB ID must be a number greater than 0');
|
||||
}
|
||||
|
||||
if (!options.type.trim()) {
|
||||
throw new Error('Must provide a type for provider sources');
|
||||
}
|
||||
|
||||
if (options.type !== 'movie' && options.type !== 'show') {
|
||||
throw new Error("Invalid media type. Must be either 'movie' or 'show'");
|
||||
}
|
||||
|
||||
if (options.type === 'show') {
|
||||
if (!options.season.trim()) {
|
||||
throw new Error('Must provide a season number for TV shows');
|
||||
}
|
||||
|
||||
if (!options.episode.trim()) {
|
||||
throw new Error('Must provide an episode number for TV shows');
|
||||
}
|
||||
|
||||
if (Number.isNaN(Number(options.season)) || Number(options.season) <= 0) {
|
||||
throw new Error('Season number must be a number greater than 0');
|
||||
}
|
||||
|
||||
if (Number.isNaN(Number(options.episode)) || Number(options.episode) <= 0) {
|
||||
throw new Error('Episode number must be a number greater than 0');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let fetcher;
|
||||
|
||||
if (options.fetcher === 'native') {
|
||||
fetcher = makeStandardFetcher(fetch);
|
||||
} else {
|
||||
fetcher = makeStandardFetcher(nodeFetch);
|
||||
}
|
||||
|
||||
const providerOptions: ProviderMakerOptions = {
|
||||
fetcher,
|
||||
target: targets.ANY,
|
||||
};
|
||||
|
||||
return {
|
||||
providerOptions,
|
||||
options,
|
||||
source,
|
||||
};
|
||||
}
|
|
@ -0,0 +1,93 @@
|
|||
import { ProviderControls, makeControls } from '@/entrypoint/controls';
|
||||
import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers';
|
||||
import { Targets, getTargetFeatures } from '@/entrypoint/utils/targets';
|
||||
import { Fetcher } from '@/fetchers/types';
|
||||
import { Embed, Sourcerer } from '@/providers/base';
|
||||
import { getProviders } from '@/providers/get';
|
||||
|
||||
export type ProviderBuilder = {
|
||||
setTarget(target: Targets): ProviderBuilder;
|
||||
setFetcher(fetcher: Fetcher): ProviderBuilder;
|
||||
setProxiedFetcher(fetcher: Fetcher): ProviderBuilder;
|
||||
addSource(scraper: Sourcerer): ProviderBuilder;
|
||||
addSource(name: string): ProviderBuilder;
|
||||
addEmbed(scraper: Embed): ProviderBuilder;
|
||||
addEmbed(name: string): ProviderBuilder;
|
||||
addBuiltinProviders(): ProviderBuilder;
|
||||
enableConsistentIpForRequests(): ProviderBuilder;
|
||||
build(): ProviderControls;
|
||||
};
|
||||
|
||||
export function buildProviders(): ProviderBuilder {
|
||||
let consistentIpForRequests = false;
|
||||
let target: Targets | null = null;
|
||||
let fetcher: Fetcher | null = null;
|
||||
let proxiedFetcher: Fetcher | null = null;
|
||||
const embeds: Embed[] = [];
|
||||
const sources: Sourcerer[] = [];
|
||||
const builtinSources = getBuiltinSources();
|
||||
const builtinEmbeds = getBuiltinEmbeds();
|
||||
|
||||
return {
|
||||
enableConsistentIpForRequests() {
|
||||
consistentIpForRequests = true;
|
||||
return this;
|
||||
},
|
||||
setFetcher(f) {
|
||||
fetcher = f;
|
||||
return this;
|
||||
},
|
||||
setProxiedFetcher(f) {
|
||||
proxiedFetcher = f;
|
||||
return this;
|
||||
},
|
||||
setTarget(t) {
|
||||
target = t;
|
||||
return this;
|
||||
},
|
||||
addSource(input) {
|
||||
if (typeof input !== 'string') {
|
||||
sources.push(input);
|
||||
return this;
|
||||
}
|
||||
|
||||
const matchingSource = builtinSources.find((v) => v.id === input);
|
||||
if (!matchingSource) throw new Error('Source not found');
|
||||
sources.push(matchingSource);
|
||||
return this;
|
||||
},
|
||||
addEmbed(input) {
|
||||
if (typeof input !== 'string') {
|
||||
embeds.push(input);
|
||||
return this;
|
||||
}
|
||||
|
||||
const matchingEmbed = builtinEmbeds.find((v) => v.id === input);
|
||||
if (!matchingEmbed) throw new Error('Embed not found');
|
||||
embeds.push(matchingEmbed);
|
||||
return this;
|
||||
},
|
||||
addBuiltinProviders() {
|
||||
sources.push(...builtinSources);
|
||||
embeds.push(...builtinEmbeds);
|
||||
return this;
|
||||
},
|
||||
build() {
|
||||
if (!target) throw new Error('Target not set');
|
||||
if (!fetcher) throw new Error('Fetcher not set');
|
||||
const features = getTargetFeatures(target, consistentIpForRequests);
|
||||
const list = getProviders(features, {
|
||||
embeds,
|
||||
sources,
|
||||
});
|
||||
|
||||
return makeControls({
|
||||
fetcher,
|
||||
proxiedFetcher: proxiedFetcher ?? undefined,
|
||||
embeds: list.embeds,
|
||||
sources: list.sources,
|
||||
features,
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
|
@ -1,24 +1,19 @@
|
|||
import { makeFullFetcher } from '@/fetchers/common';
|
||||
import { FullScraperEvents, IndividualScraperEvents } from '@/entrypoint/utils/events';
|
||||
import { ScrapeMedia } from '@/entrypoint/utils/media';
|
||||
import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/entrypoint/utils/meta';
|
||||
import { FeatureMap } from '@/entrypoint/utils/targets';
|
||||
import { makeFetcher } from '@/fetchers/common';
|
||||
import { Fetcher } from '@/fetchers/types';
|
||||
import { FullScraperEvents, IndividualScraperEvents } from '@/main/events';
|
||||
import { scrapeIndividualEmbed, scrapeInvidualSource } from '@/main/individualRunner';
|
||||
import { ScrapeMedia } from '@/main/media';
|
||||
import { MetaOutput, getAllEmbedMetaSorted, getAllSourceMetaSorted, getSpecificId } from '@/main/meta';
|
||||
import { RunOutput, runAllProviders } from '@/main/runner';
|
||||
import { Targets, getTargetFeatures } from '@/main/targets';
|
||||
import { EmbedOutput, SourcererOutput } from '@/providers/base';
|
||||
import { getProviders } from '@/providers/get';
|
||||
import { Embed, EmbedOutput, Sourcerer, SourcererOutput } from '@/providers/base';
|
||||
import { scrapeIndividualEmbed, scrapeInvidualSource } from '@/runners/individualRunner';
|
||||
import { RunOutput, runAllProviders } from '@/runners/runner';
|
||||
|
||||
export interface ProviderBuilderOptions {
|
||||
// fetcher, every web request gets called through here
|
||||
export interface ProviderControlsInput {
|
||||
fetcher: Fetcher;
|
||||
|
||||
// proxied fetcher, if the scraper needs to access a CORS proxy. this fetcher will be called instead
|
||||
// of the normal fetcher. Defaults to the normal fetcher.
|
||||
proxiedFetcher?: Fetcher;
|
||||
|
||||
// target of where the streams will be used
|
||||
target: Targets;
|
||||
features: FeatureMap;
|
||||
sources: Sourcerer[];
|
||||
embeds: Embed[];
|
||||
}
|
||||
|
||||
export interface RunnerOptions {
|
||||
|
@ -83,13 +78,16 @@ export interface ProviderControls {
|
|||
listEmbeds(): MetaOutput[];
|
||||
}
|
||||
|
||||
export function makeProviders(ops: ProviderBuilderOptions): ProviderControls {
|
||||
const features = getTargetFeatures(ops.target);
|
||||
const list = getProviders(features);
|
||||
export function makeControls(ops: ProviderControlsInput): ProviderControls {
|
||||
const list = {
|
||||
embeds: ops.embeds,
|
||||
sources: ops.sources,
|
||||
};
|
||||
|
||||
const providerRunnerOps = {
|
||||
features,
|
||||
fetcher: makeFullFetcher(ops.fetcher),
|
||||
proxiedFetcher: makeFullFetcher(ops.proxiedFetcher ?? ops.fetcher),
|
||||
features: ops.features,
|
||||
fetcher: makeFetcher(ops.fetcher),
|
||||
proxiedFetcher: makeFetcher(ops.proxiedFetcher ?? ops.fetcher),
|
||||
};
|
||||
|
||||
return {
|
|
@ -0,0 +1,37 @@
|
|||
import { makeControls } from '@/entrypoint/controls';
|
||||
import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers';
|
||||
import { Targets, getTargetFeatures } from '@/entrypoint/utils/targets';
|
||||
import { Fetcher } from '@/fetchers/types';
|
||||
import { getProviders } from '@/providers/get';
|
||||
|
||||
export interface ProviderMakerOptions {
|
||||
// fetcher, every web request gets called through here
|
||||
fetcher: Fetcher;
|
||||
|
||||
// proxied fetcher, if the scraper needs to access a CORS proxy. this fetcher will be called instead
|
||||
// of the normal fetcher. Defaults to the normal fetcher.
|
||||
proxiedFetcher?: Fetcher;
|
||||
|
||||
// target of where the streams will be used
|
||||
target: Targets;
|
||||
|
||||
// Set this to true, if the requests will have the same IP as
|
||||
// the device that the stream will be played on
|
||||
consistentIpForRequests?: boolean;
|
||||
}
|
||||
|
||||
export function makeProviders(ops: ProviderMakerOptions) {
|
||||
const features = getTargetFeatures(ops.target, ops.consistentIpForRequests ?? false);
|
||||
const list = getProviders(features, {
|
||||
embeds: getBuiltinEmbeds(),
|
||||
sources: getBuiltinSources(),
|
||||
});
|
||||
|
||||
return makeControls({
|
||||
embeds: list.embeds,
|
||||
sources: list.sources,
|
||||
features,
|
||||
fetcher: ops.fetcher,
|
||||
proxiedFetcher: ops.proxiedFetcher,
|
||||
});
|
||||
}
|
|
@ -0,0 +1,10 @@
|
|||
import { gatherAllEmbeds, gatherAllSources } from '@/providers/all';
|
||||
import { Embed, Sourcerer } from '@/providers/base';
|
||||
|
||||
export function getBuiltinSources(): Sourcerer[] {
|
||||
return gatherAllSources();
|
||||
}
|
||||
|
||||
export function getBuiltinEmbeds(): Embed[] {
|
||||
return gatherAllEmbeds();
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
import { MediaTypes } from '@/main/media';
|
||||
import { MediaTypes } from '@/entrypoint/utils/media';
|
||||
import { Embed, Sourcerer } from '@/providers/base';
|
||||
import { ProviderList } from '@/providers/get';
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
export const flags = {
|
||||
// CORS are set to allow any origin
|
||||
CORS_ALLOWED: 'cors-allowed',
|
||||
|
||||
// the stream is locked on IP, so only works if
|
||||
// request maker is same as player (not compatible with proxies)
|
||||
IP_LOCKED: 'ip-locked',
|
||||
} as const;
|
||||
|
||||
export type Flags = (typeof flags)[keyof typeof flags];
|
||||
|
||||
export const targets = {
|
||||
// browser with CORS restrictions
|
||||
BROWSER: 'browser',
|
||||
|
||||
// browser, but no CORS restrictions through a browser extension
|
||||
BROWSER_EXTENSION: 'browser-extension',
|
||||
|
||||
// native app, so no restrictions in what can be played
|
||||
NATIVE: 'native',
|
||||
|
||||
// any target, no target restrictions
|
||||
ANY: 'any',
|
||||
} as const;
|
||||
|
||||
export type Targets = (typeof targets)[keyof typeof targets];
|
||||
|
||||
export type FeatureMap = {
|
||||
requires: Flags[];
|
||||
disallowed: Flags[];
|
||||
};
|
||||
|
||||
export const targetToFeatures: Record<Targets, FeatureMap> = {
|
||||
browser: {
|
||||
requires: [flags.CORS_ALLOWED],
|
||||
disallowed: [],
|
||||
},
|
||||
'browser-extension': {
|
||||
requires: [],
|
||||
disallowed: [],
|
||||
},
|
||||
native: {
|
||||
requires: [],
|
||||
disallowed: [],
|
||||
},
|
||||
any: {
|
||||
requires: [],
|
||||
disallowed: [],
|
||||
},
|
||||
};
|
||||
|
||||
export function getTargetFeatures(target: Targets, consistentIpForRequests: boolean): FeatureMap {
|
||||
const features = targetToFeatures[target];
|
||||
if (!consistentIpForRequests) features.disallowed.push(flags.IP_LOCKED);
|
||||
return features;
|
||||
}
|
||||
|
||||
export function flagsAllowedInFeatures(features: FeatureMap, inputFlags: Flags[]): boolean {
|
||||
const hasAllFlags = features.requires.every((v) => inputFlags.includes(v));
|
||||
if (!hasAllFlags) return false;
|
||||
const hasDisallowedFlag = features.disallowed.some((v) => inputFlags.includes(v));
|
||||
if (hasDisallowedFlag) return false;
|
||||
return true;
|
||||
}
|
|
@ -26,15 +26,19 @@ export function makeFullUrl(url: string, ops?: FullUrlOptions): string {
|
|||
return parsedUrl.toString();
|
||||
}
|
||||
|
||||
export function makeFullFetcher(fetcher: Fetcher): UseableFetcher {
|
||||
return (url, ops) => {
|
||||
export function makeFetcher(fetcher: Fetcher): UseableFetcher {
|
||||
const newFetcher = (url: string, ops?: FetcherOptions) => {
|
||||
return fetcher(url, {
|
||||
headers: ops?.headers ?? {},
|
||||
method: ops?.method ?? 'GET',
|
||||
query: ops?.query ?? {},
|
||||
baseUrl: ops?.baseUrl ?? '',
|
||||
readHeaders: ops?.readHeaders ?? [],
|
||||
body: ops?.body,
|
||||
returnRaw: ops?.returnRaw ?? false,
|
||||
});
|
||||
};
|
||||
const output: UseableFetcher = async (url, ops) => (await newFetcher(url, ops)).body;
|
||||
output.full = newFetcher;
|
||||
return output;
|
||||
}
|
||||
|
|
|
@ -11,13 +11,17 @@ export type FetchOps = {
|
|||
|
||||
export type FetchHeaders = {
|
||||
get(key: string): string | null;
|
||||
set(key: string, value: string): void;
|
||||
};
|
||||
|
||||
export type FetchReply = {
|
||||
text(): Promise<string>;
|
||||
json(): Promise<any>;
|
||||
extraHeaders?: FetchHeaders;
|
||||
extraUrl?: string;
|
||||
headers: FetchHeaders;
|
||||
url: string;
|
||||
status: number;
|
||||
};
|
||||
|
||||
export type FetchLike = (url: string, ops?: FetchOps | undefined) => Promise<FetchReply>;
|
||||
|
|
|
@ -9,9 +9,28 @@ const headerMap: Record<string, string> = {
|
|||
origin: 'X-Origin',
|
||||
};
|
||||
|
||||
const responseHeaderMap: Record<string, string> = {
|
||||
'x-set-cookie': 'Set-Cookie',
|
||||
};
|
||||
|
||||
export function makeSimpleProxyFetcher(proxyUrl: string, f: FetchLike): Fetcher {
|
||||
const fetcher = makeStandardFetcher(f);
|
||||
const proxiedFetch: Fetcher = async (url, ops) => {
|
||||
const fetcher = makeStandardFetcher(async (a, b) => {
|
||||
const res = await f(a, b);
|
||||
|
||||
// set extra headers that cant normally be accessed
|
||||
res.extraHeaders = new Headers();
|
||||
Object.entries(responseHeaderMap).forEach((entry) => {
|
||||
const value = res.headers.get(entry[0]);
|
||||
if (!value) return;
|
||||
res.extraHeaders?.set(entry[0].toLowerCase(), value);
|
||||
});
|
||||
|
||||
// set correct final url
|
||||
res.extraUrl = res.headers.get('X-Final-Destination') ?? res.url;
|
||||
return res;
|
||||
});
|
||||
|
||||
const fullUrl = makeFullUrl(url, ops);
|
||||
|
||||
const headerEntries = Object.entries(ops.headers).map((entry) => {
|
||||
|
|
|
@ -1,8 +1,20 @@
|
|||
import { serializeBody } from '@/fetchers/body';
|
||||
import { makeFullUrl } from '@/fetchers/common';
|
||||
import { FetchLike } from '@/fetchers/fetch';
|
||||
import { FetchLike, FetchReply } from '@/fetchers/fetch';
|
||||
import { Fetcher } from '@/fetchers/types';
|
||||
|
||||
function getHeaders(list: string[], res: FetchReply): Headers {
|
||||
const output = new Headers();
|
||||
list.forEach((header) => {
|
||||
const realHeader = header.toLowerCase();
|
||||
const value = res.headers.get(realHeader);
|
||||
const extraValue = res.extraHeaders?.get(realHeader);
|
||||
if (!value) return;
|
||||
output.set(realHeader, extraValue ?? value);
|
||||
});
|
||||
return output;
|
||||
}
|
||||
|
||||
export function makeStandardFetcher(f: FetchLike): Fetcher {
|
||||
const normalFetch: Fetcher = async (url, ops) => {
|
||||
const fullUrl = makeFullUrl(url, ops);
|
||||
|
@ -21,9 +33,17 @@ export function makeStandardFetcher(f: FetchLike): Fetcher {
|
|||
return res;
|
||||
}
|
||||
|
||||
let body: any;
|
||||
const isJson = res.headers.get('content-type')?.includes('application/json');
|
||||
if (isJson) return res.json();
|
||||
return res.text();
|
||||
if (isJson) body = await res.json();
|
||||
else body = await res.text();
|
||||
|
||||
return {
|
||||
body,
|
||||
finalUrl: res.extraUrl ?? res.url,
|
||||
headers: getHeaders(ops.readHeaders, res),
|
||||
statusCode: res.status,
|
||||
};
|
||||
};
|
||||
|
||||
return normalFetch;
|
||||
|
|
|
@ -5,10 +5,13 @@ export type FetcherOptions = {
|
|||
headers?: Record<string, string>;
|
||||
query?: Record<string, string>;
|
||||
method?: 'HEAD' | 'GET' | 'POST';
|
||||
readHeaders?: string[];
|
||||
body?: Record<string, any> | string | FormData | URLSearchParams;
|
||||
returnRaw?: boolean;
|
||||
};
|
||||
|
||||
// Version of the options that always has the defaults set
|
||||
// This is to make making fetchers yourself easier
|
||||
export type DefaultedFetcherOptions = {
|
||||
baseUrl?: string;
|
||||
body?: Record<string, any> | string | FormData;
|
||||
|
@ -16,13 +19,23 @@ export type DefaultedFetcherOptions = {
|
|||
query: Record<string, string>;
|
||||
method: 'HEAD' | 'GET' | 'POST';
|
||||
returnRaw: boolean;
|
||||
readHeaders: string[];
|
||||
};
|
||||
|
||||
export type FetcherResponse<T = any> = {
|
||||
statusCode: number;
|
||||
headers: Headers;
|
||||
finalUrl: string;
|
||||
body: T;
|
||||
};
|
||||
|
||||
// This is the version that will be inputted by library users
|
||||
export type Fetcher<T = any> = {
|
||||
(url: string, ops: DefaultedFetcherOptions): Promise<T>;
|
||||
(url: string, ops: DefaultedFetcherOptions): Promise<FetcherResponse<T>>;
|
||||
};
|
||||
|
||||
// this feature has some quality of life features
|
||||
// This is the version that scrapers will be interacting with
|
||||
export type UseableFetcher<T = any> = {
|
||||
(url: string, ops?: FetcherOptions): Promise<T>;
|
||||
full: (url: string, ops?: FetcherOptions) => Promise<FetcherResponse<T>>;
|
||||
};
|
||||
|
|
28
src/index.ts
28
src/index.ts
|
@ -1,19 +1,19 @@
|
|||
export type { EmbedOutput, SourcererOutput } from '@/providers/base';
|
||||
export type { RunOutput } from '@/main/runner';
|
||||
export type { MetaOutput } from '@/main/meta';
|
||||
export type { FullScraperEvents } from '@/main/events';
|
||||
export type { Targets, Flags } from '@/main/targets';
|
||||
export type { MediaTypes, ShowMedia, ScrapeMedia, MovieMedia } from '@/main/media';
|
||||
export type {
|
||||
ProviderBuilderOptions,
|
||||
ProviderControls,
|
||||
RunnerOptions,
|
||||
EmbedRunnerOptions,
|
||||
SourceRunnerOptions,
|
||||
} from '@/main/builder';
|
||||
export type { Stream, StreamFile, FileBasedStream, HlsBasedStream, Qualities } from '@/providers/streams';
|
||||
export type { Fetcher, FetcherOptions, FetcherResponse } from '@/fetchers/types';
|
||||
export type { RunOutput } from '@/runners/runner';
|
||||
export type { MetaOutput } from '@/entrypoint/utils/meta';
|
||||
export type { FullScraperEvents } from '@/entrypoint/utils/events';
|
||||
export type { Targets, Flags } from '@/entrypoint/utils/targets';
|
||||
export type { MediaTypes, ShowMedia, ScrapeMedia, MovieMedia } from '@/entrypoint/utils/media';
|
||||
export type { ProviderControls, RunnerOptions, EmbedRunnerOptions, SourceRunnerOptions } from '@/entrypoint/controls';
|
||||
export type { ProviderBuilder } from '@/entrypoint/builder';
|
||||
export type { ProviderMakerOptions } from '@/entrypoint/declare';
|
||||
|
||||
export { NotFoundError } from '@/utils/errors';
|
||||
export { makeProviders } from '@/main/builder';
|
||||
export { makeProviders } from '@/entrypoint/declare';
|
||||
export { buildProviders } from '@/entrypoint/builder';
|
||||
export { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers';
|
||||
export { makeStandardFetcher } from '@/fetchers/standardFetch';
|
||||
export { makeSimpleProxyFetcher } from '@/fetchers/simpleProxy';
|
||||
export { flags, targets } from '@/main/targets';
|
||||
export { flags, targets } from '@/entrypoint/utils/targets';
|
||||
|
|
|
@ -1,40 +0,0 @@
|
|||
export const flags = {
|
||||
NO_CORS: 'no-cors',
|
||||
IP_LOCKED: 'ip-locked',
|
||||
} as const;
|
||||
|
||||
export type Flags = (typeof flags)[keyof typeof flags];
|
||||
|
||||
export const targets = {
|
||||
BROWSER: 'browser',
|
||||
NATIVE: 'native',
|
||||
ALL: 'all',
|
||||
} as const;
|
||||
|
||||
export type Targets = (typeof targets)[keyof typeof targets];
|
||||
|
||||
export type FeatureMap = {
|
||||
requires: readonly Flags[];
|
||||
};
|
||||
|
||||
export const targetToFeatures: Record<Targets, FeatureMap> = {
|
||||
browser: {
|
||||
requires: [flags.NO_CORS],
|
||||
},
|
||||
native: {
|
||||
requires: [],
|
||||
},
|
||||
all: {
|
||||
requires: [],
|
||||
},
|
||||
} as const;
|
||||
|
||||
export function getTargetFeatures(target: Targets): FeatureMap {
|
||||
return targetToFeatures[target];
|
||||
}
|
||||
|
||||
export function flagsAllowedInFeatures(features: FeatureMap, inputFlags: Flags[]): boolean {
|
||||
const hasAllFlags = features.requires.every((v) => inputFlags.includes(v));
|
||||
if (!hasAllFlags) return false;
|
||||
return true;
|
||||
}
|
|
@ -1,7 +1,9 @@
|
|||
import { Flags } from '@/main/targets';
|
||||
import { Flags } from '@/entrypoint/utils/targets';
|
||||
import { Stream } from '@/providers/streams';
|
||||
import { EmbedScrapeContext, MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
|
||||
|
||||
export type MediaScraperTypes = 'show' | 'movie';
|
||||
|
||||
export type SourcererEmbed = {
|
||||
embedId: string;
|
||||
url: string;
|
||||
|
@ -10,10 +12,10 @@ export type SourcererEmbed = {
|
|||
|
||||
export type SourcererOutput = {
|
||||
embeds: SourcererEmbed[];
|
||||
stream?: Stream;
|
||||
stream?: Stream[];
|
||||
};
|
||||
|
||||
export type Sourcerer = {
|
||||
export type SourcererOptions = {
|
||||
id: string;
|
||||
name: string; // displayed in the UI
|
||||
rank: number; // the higher the number, the earlier it gets put on the queue
|
||||
|
@ -23,15 +25,29 @@ export type Sourcerer = {
|
|||
scrapeShow?: (input: ShowScrapeContext) => Promise<SourcererOutput>;
|
||||
};
|
||||
|
||||
export function makeSourcerer(state: Sourcerer): Sourcerer {
|
||||
return state;
|
||||
export type Sourcerer = SourcererOptions & {
|
||||
type: 'source';
|
||||
disabled: boolean;
|
||||
mediaTypes: MediaScraperTypes[];
|
||||
};
|
||||
|
||||
export function makeSourcerer(state: SourcererOptions): Sourcerer {
|
||||
const mediaTypes: MediaScraperTypes[] = [];
|
||||
if (state.scrapeMovie) mediaTypes.push('movie');
|
||||
if (state.scrapeShow) mediaTypes.push('show');
|
||||
return {
|
||||
...state,
|
||||
type: 'source',
|
||||
disabled: state.disabled ?? false,
|
||||
mediaTypes,
|
||||
};
|
||||
}
|
||||
|
||||
export type EmbedOutput = {
|
||||
stream: Stream;
|
||||
stream: Stream[];
|
||||
};
|
||||
|
||||
export type Embed = {
|
||||
export type EmbedOptions = {
|
||||
id: string;
|
||||
name: string; // displayed in the UI
|
||||
rank: number; // the higher the number, the earlier it gets put on the queue
|
||||
|
@ -39,6 +55,17 @@ export type Embed = {
|
|||
scrape: (input: EmbedScrapeContext) => Promise<EmbedOutput>;
|
||||
};
|
||||
|
||||
export function makeEmbed(state: Embed): Embed {
|
||||
return state;
|
||||
export type Embed = EmbedOptions & {
|
||||
type: 'embed';
|
||||
disabled: boolean;
|
||||
mediaTypes: undefined;
|
||||
};
|
||||
|
||||
export function makeEmbed(state: EmbedOptions): Embed {
|
||||
return {
|
||||
...state,
|
||||
type: 'embed',
|
||||
disabled: state.disabled ?? false,
|
||||
mediaTypes: undefined,
|
||||
};
|
||||
}
|
||||
|
|
|
@ -8,6 +8,7 @@ export type CaptionType = keyof typeof captionTypes;
|
|||
|
||||
export type Caption = {
|
||||
type: CaptionType;
|
||||
id: string; // only unique per stream
|
||||
url: string;
|
||||
hasCorsRestrictions: boolean;
|
||||
language: string;
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import { MediaTypes } from '@/main/media';
|
||||
import { MediaTypes } from '@/entrypoint/utils/media';
|
||||
|
||||
export const febBoxBase = `https://www.febbox.com`;
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import { MediaTypes } from '@/main/media';
|
||||
import { MediaTypes } from '@/entrypoint/utils/media';
|
||||
import { FebboxFileList, febBoxBase } from '@/providers/embeds/febbox/common';
|
||||
import { EmbedScrapeContext } from '@/utils/context';
|
||||
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
import { MediaTypes } from '@/main/media';
|
||||
import { flags } from '@/main/targets';
|
||||
import { MediaTypes } from '@/entrypoint/utils/media';
|
||||
import { flags } from '@/entrypoint/utils/targets';
|
||||
import { makeEmbed } from '@/providers/base';
|
||||
import { parseInputUrl } from '@/providers/embeds/febbox/common';
|
||||
import { getStreams } from '@/providers/embeds/febbox/fileList';
|
||||
|
@ -36,12 +36,15 @@ export const febboxHlsScraper = makeEmbed({
|
|||
ctx.progress(70);
|
||||
|
||||
return {
|
||||
stream: {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
flags: [flags.NO_CORS],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
captions: await getSubtitles(ctx, id, firstStream.fid, type as MediaTypes, season, episode),
|
||||
playlist: `https://www.febbox.com/hls/main/${firstStream.oss_fid}.m3u8`,
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
});
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import { flags } from '@/main/targets';
|
||||
import { flags } from '@/entrypoint/utils/targets';
|
||||
import { makeEmbed } from '@/providers/base';
|
||||
import { parseInputUrl } from '@/providers/embeds/febbox/common';
|
||||
import { getStreamQualities } from '@/providers/embeds/febbox/qualities';
|
||||
|
@ -39,12 +39,15 @@ export const febboxMp4Scraper = makeEmbed({
|
|||
ctx.progress(70);
|
||||
|
||||
return {
|
||||
stream: {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
captions: await getSubtitles(ctx, id, fid, type, episode, season),
|
||||
qualities,
|
||||
type: 'file',
|
||||
flags: [flags.NO_CORS],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
});
|
||||
|
|
|
@ -54,6 +54,7 @@ export async function getSubtitles(
|
|||
if (!validCode) return;
|
||||
|
||||
output.push({
|
||||
id: subtitleFilePath,
|
||||
language: subtitle.lang,
|
||||
hasCorsRestrictions: true,
|
||||
type: subtitleType,
|
||||
|
|
|
@ -33,7 +33,9 @@ export const mixdropScraper = makeEmbed({
|
|||
const url = link[1];
|
||||
|
||||
return {
|
||||
stream: {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'file',
|
||||
flags: [],
|
||||
captions: [],
|
||||
|
@ -48,6 +50,7 @@ export const mixdropScraper = makeEmbed({
|
|||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
});
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import { flags } from '@/main/targets';
|
||||
import { flags } from '@/entrypoint/utils/targets';
|
||||
import { makeEmbed } from '@/providers/base';
|
||||
|
||||
export const mp4uploadScraper = makeEmbed({
|
||||
|
@ -15,9 +15,11 @@ export const mp4uploadScraper = makeEmbed({
|
|||
if (!streamUrl) throw new Error('Stream url not found in embed code');
|
||||
|
||||
return {
|
||||
stream: {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'file',
|
||||
flags: [flags.NO_CORS],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
captions: [],
|
||||
qualities: {
|
||||
'1080': {
|
||||
|
@ -26,6 +28,7 @@ export const mp4uploadScraper = makeEmbed({
|
|||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
});
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import { load } from 'cheerio';
|
||||
|
||||
import { flags } from '@/main/targets';
|
||||
import { flags } from '@/entrypoint/utils/targets';
|
||||
import { makeEmbed } from '@/providers/base';
|
||||
|
||||
type DPlayerSourcesResponse = {
|
||||
|
@ -57,12 +57,15 @@ export const smashyStreamDScraper = makeEmbed({
|
|||
);
|
||||
|
||||
return {
|
||||
stream: {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
playlist: playlistRes,
|
||||
type: 'hls',
|
||||
flags: [flags.NO_CORS],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
captions: [],
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
});
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import { flags } from '@/main/targets';
|
||||
import { flags } from '@/entrypoint/utils/targets';
|
||||
import { makeEmbed } from '@/providers/base';
|
||||
import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions';
|
||||
|
||||
|
@ -30,6 +30,7 @@ export const smashyStreamFScraper = makeEmbed({
|
|||
const captionType = getCaptionTypeFromUrl(url);
|
||||
if (!languageCode || !captionType) return null;
|
||||
return {
|
||||
id: url,
|
||||
url: url.replace(',', ''),
|
||||
language: languageCode,
|
||||
type: captionType,
|
||||
|
@ -42,12 +43,15 @@ export const smashyStreamFScraper = makeEmbed({
|
|||
.filter((x): x is Caption => x !== null) ?? [];
|
||||
|
||||
return {
|
||||
stream: {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
playlist: res.sourceUrls[0],
|
||||
type: 'hls',
|
||||
flags: [flags.NO_CORS],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
captions,
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
});
|
||||
|
|
|
@ -3,7 +3,7 @@ import Base64 from 'crypto-js/enc-base64';
|
|||
import Utf8 from 'crypto-js/enc-utf8';
|
||||
import FormData from 'form-data';
|
||||
|
||||
import { flags } from '@/main/targets';
|
||||
import { flags } from '@/entrypoint/utils/targets';
|
||||
import { makeEmbed } from '@/providers/base';
|
||||
import { StreamFile } from '@/providers/streams';
|
||||
import { EmbedScrapeContext } from '@/utils/context';
|
||||
|
@ -155,12 +155,15 @@ export const streamsbScraper = makeEmbed({
|
|||
}, {} as Record<string, StreamFile>);
|
||||
|
||||
return {
|
||||
stream: {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'file',
|
||||
flags: [flags.NO_CORS],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
qualities,
|
||||
captions: [],
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
});
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import crypto from 'crypto-js';
|
||||
|
||||
import { flags } from '@/main/targets';
|
||||
import { flags } from '@/entrypoint/utils/targets';
|
||||
import { makeEmbed } from '@/providers/base';
|
||||
import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions';
|
||||
|
||||
|
@ -110,6 +110,7 @@ export const upcloudScraper = makeEmbed({
|
|||
const language = labelToLanguageCode(track.label);
|
||||
if (!language) return;
|
||||
captions.push({
|
||||
id: track.file,
|
||||
language,
|
||||
hasCorsRestrictions: false,
|
||||
type,
|
||||
|
@ -118,12 +119,15 @@ export const upcloudScraper = makeEmbed({
|
|||
});
|
||||
|
||||
return {
|
||||
stream: {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: sources.file,
|
||||
flags: [flags.NO_CORS],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
captions,
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
});
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import * as unpacker from 'unpacker';
|
||||
|
||||
import { flags } from '@/main/targets';
|
||||
import { flags } from '@/entrypoint/utils/targets';
|
||||
import { makeEmbed } from '@/providers/base';
|
||||
|
||||
const packedRegex = /(eval\(function\(p,a,c,k,e,d\).*\)\)\))/;
|
||||
|
@ -21,12 +21,15 @@ export const upstreamScraper = makeEmbed({
|
|||
|
||||
if (link) {
|
||||
return {
|
||||
stream: {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
type: 'hls',
|
||||
playlist: link[1],
|
||||
flags: [flags.NO_CORS],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
captions: [],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
import { FeatureMap, flagsAllowedInFeatures } from '@/main/targets';
|
||||
import { gatherAllEmbeds, gatherAllSources } from '@/providers/all';
|
||||
import { FeatureMap, flagsAllowedInFeatures } from '@/entrypoint/utils/targets';
|
||||
import { Embed, Sourcerer } from '@/providers/base';
|
||||
import { hasDuplicates } from '@/utils/predicates';
|
||||
|
||||
|
@ -8,9 +7,9 @@ export interface ProviderList {
|
|||
embeds: Embed[];
|
||||
}
|
||||
|
||||
export function getProviders(features: FeatureMap): ProviderList {
|
||||
const sources = gatherAllSources().filter((v) => !v?.disabled);
|
||||
const embeds = gatherAllEmbeds().filter((v) => !v?.disabled);
|
||||
export function getProviders(features: FeatureMap, list: ProviderList): ProviderList {
|
||||
const sources = list.sources.filter((v) => !v?.disabled);
|
||||
const embeds = list.embeds.filter((v) => !v?.disabled);
|
||||
const combined = [...sources, ...embeds];
|
||||
|
||||
const anyDuplicateId = hasDuplicates(combined.map((v) => v.id));
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import { flags } from '@/main/targets';
|
||||
import { flags } from '@/entrypoint/utils/targets';
|
||||
import { makeSourcerer } from '@/providers/base';
|
||||
import { upcloudScraper } from '@/providers/embeds/upcloud';
|
||||
import { getFlixhqMovieSources, getFlixhqShowSources, getFlixhqSourceDetails } from '@/providers/sources/flixhq/scrape';
|
||||
|
@ -9,7 +9,7 @@ export const flixhqScraper = makeSourcerer({
|
|||
id: 'flixhq',
|
||||
name: 'FlixHQ',
|
||||
rank: 100,
|
||||
flags: [flags.NO_CORS],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
async scrapeMovie(ctx) {
|
||||
const id = await getFlixhqId(ctx, ctx.media);
|
||||
if (!id) throw new NotFoundError('no search results match');
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import { load } from 'cheerio';
|
||||
|
||||
import { MovieMedia, ShowMedia } from '@/main/media';
|
||||
import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media';
|
||||
import { flixHqBase } from '@/providers/sources/flixhq/common';
|
||||
import { ScrapeContext } from '@/utils/context';
|
||||
import { NotFoundError } from '@/utils/errors';
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import { load } from 'cheerio';
|
||||
|
||||
import { MovieMedia, ShowMedia } from '@/main/media';
|
||||
import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media';
|
||||
import { flixHqBase } from '@/providers/sources/flixhq/common';
|
||||
import { compareMedia, compareTitle } from '@/utils/compare';
|
||||
import { ScrapeContext } from '@/utils/context';
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import { load } from 'cheerio';
|
||||
|
||||
import { flags } from '@/main/targets';
|
||||
import { flags } from '@/entrypoint/utils/targets';
|
||||
import { makeSourcerer } from '@/providers/base';
|
||||
import { upcloudScraper } from '@/providers/embeds/upcloud';
|
||||
import { NotFoundError } from '@/utils/errors';
|
||||
|
@ -13,7 +13,7 @@ export const goMoviesScraper = makeSourcerer({
|
|||
id: 'gomovies',
|
||||
name: 'GOmovies',
|
||||
rank: 110,
|
||||
flags: [flags.NO_CORS],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
async scrapeShow(ctx) {
|
||||
const search = await ctx.proxiedFetcher<string>(`/ajax/search`, {
|
||||
method: 'POST',
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import { load } from 'cheerio';
|
||||
|
||||
import { flags } from '@/main/targets';
|
||||
import { flags } from '@/entrypoint/utils/targets';
|
||||
import { makeSourcerer } from '@/providers/base';
|
||||
import { NotFoundError } from '@/utils/errors';
|
||||
|
||||
|
@ -13,7 +13,7 @@ export const kissAsianScraper = makeSourcerer({
|
|||
id: 'kissasian',
|
||||
name: 'KissAsian',
|
||||
rank: 130,
|
||||
flags: [flags.NO_CORS],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
disabled: true,
|
||||
|
||||
async scrapeShow(ctx) {
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import { flags } from '@/main/targets';
|
||||
import { flags } from '@/entrypoint/utils/targets';
|
||||
import { SourcererOutput, makeSourcerer } from '@/providers/base';
|
||||
import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
|
||||
import { NotFoundError } from '@/utils/errors';
|
||||
|
@ -17,12 +17,15 @@ async function universalScraper(ctx: MovieScrapeContext | ShowScrapeContext): Pr
|
|||
|
||||
return {
|
||||
embeds: [],
|
||||
stream: {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
playlist: videoUrl,
|
||||
type: 'hls',
|
||||
flags: [flags.IP_LOCKED],
|
||||
captions: [],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import { MovieMedia } from '@/main/media';
|
||||
import { MovieMedia } from '@/entrypoint/utils/media';
|
||||
|
||||
// ! Types
|
||||
interface BaseConfig {
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import { MovieMedia, ShowMedia } from '@/main/media';
|
||||
import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media';
|
||||
import { compareMedia } from '@/utils/compare';
|
||||
import { ScrapeContext } from '@/utils/context';
|
||||
import { NotFoundError } from '@/utils/errors';
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import { MovieMedia, ShowMedia } from '@/main/media';
|
||||
import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media';
|
||||
import { ScrapeContext } from '@/utils/context';
|
||||
|
||||
import { StreamsDataResult } from './type';
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import { flags } from '@/main/targets';
|
||||
import { flags } from '@/entrypoint/utils/targets';
|
||||
import { makeSourcerer } from '@/providers/base';
|
||||
import { NotFoundError } from '@/utils/errors';
|
||||
|
||||
|
@ -8,7 +8,7 @@ export const remotestreamScraper = makeSourcerer({
|
|||
id: 'remotestream',
|
||||
name: 'Remote Stream',
|
||||
rank: 55,
|
||||
flags: [flags.NO_CORS],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
async scrapeShow(ctx) {
|
||||
const seasonNumber = ctx.media.season.number;
|
||||
const episodeNumber = ctx.media.episode.number;
|
||||
|
@ -22,12 +22,15 @@ export const remotestreamScraper = makeSourcerer({
|
|||
|
||||
return {
|
||||
embeds: [],
|
||||
stream: {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
captions: [],
|
||||
playlist: playlistLink,
|
||||
type: 'hls',
|
||||
flags: [flags.NO_CORS],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
async scrapeMovie(ctx) {
|
||||
|
@ -40,12 +43,15 @@ export const remotestreamScraper = makeSourcerer({
|
|||
|
||||
return {
|
||||
embeds: [],
|
||||
stream: {
|
||||
stream: [
|
||||
{
|
||||
id: 'primary',
|
||||
captions: [],
|
||||
playlist: playlistLink,
|
||||
type: 'hls',
|
||||
flags: [flags.NO_CORS],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
});
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import { flags } from '@/main/targets';
|
||||
import { flags } from '@/entrypoint/utils/targets';
|
||||
import { SourcererOutput, makeSourcerer } from '@/providers/base';
|
||||
import { febboxHlsScraper } from '@/providers/embeds/febbox/hls';
|
||||
import { febboxMp4Scraper } from '@/providers/embeds/febbox/mp4';
|
||||
|
@ -47,7 +47,7 @@ export const showboxScraper = makeSourcerer({
|
|||
id: 'showbox',
|
||||
name: 'Showbox',
|
||||
rank: 300,
|
||||
flags: [flags.NO_CORS],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
scrapeShow: comboScraper,
|
||||
scrapeMovie: comboScraper,
|
||||
});
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import { load } from 'cheerio';
|
||||
|
||||
import { flags } from '@/main/targets';
|
||||
import { flags } from '@/entrypoint/utils/targets';
|
||||
import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base';
|
||||
import { smashyStreamDScraper } from '@/providers/embeds/smashystream/dued';
|
||||
import { smashyStreamFScraper } from '@/providers/embeds/smashystream/video1';
|
||||
|
@ -58,7 +58,7 @@ export const smashyStreamScraper = makeSourcerer({
|
|||
id: 'smashystream',
|
||||
name: 'SmashyStream',
|
||||
rank: 70,
|
||||
flags: [flags.NO_CORS],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
scrapeMovie: universalScraper,
|
||||
scrapeShow: universalScraper,
|
||||
});
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import { flags } from '@/main/targets';
|
||||
import { flags } from '@/entrypoint/utils/targets';
|
||||
import { makeSourcerer } from '@/providers/base';
|
||||
import { scrapeMovie } from '@/providers/sources/zoechip/scrape-movie';
|
||||
import { scrapeShow } from '@/providers/sources/zoechip/scrape-show';
|
||||
|
@ -7,7 +7,7 @@ export const zoechipScraper = makeSourcerer({
|
|||
id: 'zoechip',
|
||||
name: 'ZoeChip',
|
||||
rank: 200,
|
||||
flags: [flags.NO_CORS],
|
||||
flags: [flags.CORS_ALLOWED],
|
||||
scrapeMovie,
|
||||
scrapeShow,
|
||||
});
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import { load } from 'cheerio';
|
||||
|
||||
import { ShowMedia } from '@/main/media';
|
||||
import { ShowMedia } from '@/entrypoint/utils/media';
|
||||
import { ZoeChipSourceDetails, zoeBase } from '@/providers/sources/zoechip/common';
|
||||
import { MovieScrapeContext, ScrapeContext, ShowScrapeContext } from '@/utils/context';
|
||||
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import { load } from 'cheerio';
|
||||
|
||||
import { MovieMedia, ShowMedia } from '@/main/media';
|
||||
import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media';
|
||||
import { zoeBase } from '@/providers/sources/zoechip/common';
|
||||
import { compareMedia } from '@/utils/compare';
|
||||
import { ScrapeContext } from '@/utils/context';
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
import { Flags } from '@/main/targets';
|
||||
import { Flags } from '@/entrypoint/utils/targets';
|
||||
import { Caption } from '@/providers/captions';
|
||||
|
||||
export type StreamFile = {
|
||||
|
@ -9,18 +9,22 @@ export type StreamFile = {
|
|||
|
||||
export type Qualities = 'unknown' | '360' | '480' | '720' | '1080' | '4k';
|
||||
|
||||
export type FileBasedStream = {
|
||||
type: 'file';
|
||||
type StreamCommon = {
|
||||
id: string; // only unique per output
|
||||
flags: Flags[];
|
||||
qualities: Partial<Record<Qualities, StreamFile>>;
|
||||
captions: Caption[];
|
||||
headers?: Record<string, string>; // these headers HAVE to be set to watch the stream
|
||||
preferredHeaders?: Record<string, string>; // these headers are optional, would improve the stream
|
||||
};
|
||||
|
||||
export type HlsBasedStream = {
|
||||
export type FileBasedStream = StreamCommon & {
|
||||
type: 'file';
|
||||
qualities: Partial<Record<Qualities, StreamFile>>;
|
||||
};
|
||||
|
||||
export type HlsBasedStream = StreamCommon & {
|
||||
type: 'hls';
|
||||
flags: Flags[];
|
||||
playlist: string;
|
||||
captions: Caption[];
|
||||
};
|
||||
|
||||
export type Stream = FileBasedStream | HlsBasedStream;
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
import { IndividualScraperEvents } from '@/entrypoint/utils/events';
|
||||
import { ScrapeMedia } from '@/entrypoint/utils/media';
|
||||
import { FeatureMap, flagsAllowedInFeatures } from '@/entrypoint/utils/targets';
|
||||
import { UseableFetcher } from '@/fetchers/types';
|
||||
import { IndividualScraperEvents } from '@/main/events';
|
||||
import { ScrapeMedia } from '@/main/media';
|
||||
import { FeatureMap, flagsAllowedInFeatures } from '@/main/targets';
|
||||
import { EmbedOutput, SourcererOutput } from '@/providers/base';
|
||||
import { ProviderList } from '@/providers/get';
|
||||
import { ScrapeContext } from '@/utils/context';
|
||||
|
@ -50,12 +50,16 @@ export async function scrapeInvidualSource(
|
|||
media: ops.media,
|
||||
});
|
||||
|
||||
// stream doesn't satisfy the feature flags, so gets removed in output
|
||||
if (output?.stream && (!isValidStream(output.stream) || !flagsAllowedInFeatures(ops.features, output.stream.flags))) {
|
||||
output.stream = undefined;
|
||||
// filter output with only valid streams
|
||||
if (output?.stream) {
|
||||
output.stream = output.stream
|
||||
.filter((stream) => isValidStream(stream))
|
||||
.filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
|
||||
}
|
||||
|
||||
if (!output) throw new Error('output is null');
|
||||
if ((!output.stream || output.stream.length === 0) && output.embeds.length === 0)
|
||||
throw new NotFoundError('No streams found');
|
||||
return output;
|
||||
}
|
||||
|
||||
|
@ -90,9 +94,10 @@ export async function scrapeIndividualEmbed(
|
|||
},
|
||||
});
|
||||
|
||||
if (!isValidStream(output.stream)) throw new NotFoundError('stream is incomplete');
|
||||
if (!flagsAllowedInFeatures(ops.features, output.stream.flags))
|
||||
throw new NotFoundError("stream doesn't satisfy target feature flags");
|
||||
output.stream = output.stream
|
||||
.filter((stream) => isValidStream(stream))
|
||||
.filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
|
||||
if (output.stream.length === 0) throw new NotFoundError('No streams found');
|
||||
|
||||
return output;
|
||||
}
|
|
@ -1,7 +1,7 @@
|
|||
import { FullScraperEvents } from '@/entrypoint/utils/events';
|
||||
import { ScrapeMedia } from '@/entrypoint/utils/media';
|
||||
import { FeatureMap, flagsAllowedInFeatures } from '@/entrypoint/utils/targets';
|
||||
import { UseableFetcher } from '@/fetchers/types';
|
||||
import { FullScraperEvents } from '@/main/events';
|
||||
import { ScrapeMedia } from '@/main/media';
|
||||
import { FeatureMap, flagsAllowedInFeatures } from '@/main/targets';
|
||||
import { EmbedOutput, SourcererOutput } from '@/providers/base';
|
||||
import { ProviderList } from '@/providers/get';
|
||||
import { Stream } from '@/providers/streams';
|
||||
|
@ -18,13 +18,13 @@ export type RunOutput = {
|
|||
|
||||
export type SourceRunOutput = {
|
||||
sourceId: string;
|
||||
stream?: Stream;
|
||||
stream: Stream[];
|
||||
embeds: [];
|
||||
};
|
||||
|
||||
export type EmbedRunOutput = {
|
||||
embedId: string;
|
||||
stream?: Stream;
|
||||
stream: Stream[];
|
||||
};
|
||||
|
||||
export type ProviderRunnerOptions = {
|
||||
|
@ -80,12 +80,14 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
|
|||
...contextBase,
|
||||
media: ops.media,
|
||||
});
|
||||
if (output?.stream && !isValidStream(output?.stream)) {
|
||||
throw new NotFoundError('stream is incomplete');
|
||||
}
|
||||
if (output?.stream && !flagsAllowedInFeatures(ops.features, output.stream.flags)) {
|
||||
throw new NotFoundError("stream doesn't satisfy target feature flags");
|
||||
if (output) {
|
||||
output.stream = (output.stream ?? [])
|
||||
.filter((stream) => isValidStream(stream))
|
||||
.filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
|
||||
}
|
||||
if (!output) throw Error('No output');
|
||||
if ((!output.stream || output.stream.length === 0) && output.embeds.length === 0)
|
||||
throw new NotFoundError('No streams found');
|
||||
} catch (err) {
|
||||
if (err instanceof NotFoundError) {
|
||||
ops.events?.update?.({
|
||||
|
@ -107,10 +109,10 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
|
|||
if (!output) throw new Error('Invalid media type');
|
||||
|
||||
// return stream is there are any
|
||||
if (output.stream) {
|
||||
if (output.stream?.[0]) {
|
||||
return {
|
||||
sourceId: s.id,
|
||||
stream: output.stream,
|
||||
stream: output.stream[0],
|
||||
};
|
||||
}
|
||||
|
||||
|
@ -144,9 +146,10 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
|
|||
...contextBase,
|
||||
url: e.url,
|
||||
});
|
||||
if (!flagsAllowedInFeatures(ops.features, embedOutput.stream.flags)) {
|
||||
throw new NotFoundError("stream doesn't satisfy target feature flags");
|
||||
}
|
||||
embedOutput.stream = embedOutput.stream
|
||||
.filter((stream) => isValidStream(stream))
|
||||
.filter((stream) => flagsAllowedInFeatures(ops.features, stream.flags));
|
||||
if (embedOutput.stream.length === 0) throw new NotFoundError('No streams found');
|
||||
} catch (err) {
|
||||
if (err instanceof NotFoundError) {
|
||||
ops.events?.update?.({
|
||||
|
@ -169,7 +172,7 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
|
|||
return {
|
||||
sourceId: s.id,
|
||||
embedId: scraper.id,
|
||||
stream: embedOutput.stream,
|
||||
stream: embedOutput.stream[0],
|
||||
};
|
||||
}
|
||||
}
|
|
@ -1,4 +1,4 @@
|
|||
import { CommonMedia } from '@/main/media';
|
||||
import { CommonMedia } from '@/entrypoint/utils/media';
|
||||
|
||||
export function normalizeTitle(title: string): string {
|
||||
return title
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media';
|
||||
import { UseableFetcher } from '@/fetchers/types';
|
||||
import { MovieMedia, ShowMedia } from '@/main/media';
|
||||
|
||||
export type ScrapeContext = {
|
||||
proxiedFetcher: <T>(...params: Parameters<UseableFetcher<T>>) => ReturnType<UseableFetcher<T>>;
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
dist
|
|
@ -0,0 +1,11 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Browser integration test</title>
|
||||
</head>
|
||||
<body>
|
||||
<script type="module" src="index.ts"></script>
|
||||
</body>
|
||||
</html>
|
|
@ -0,0 +1,8 @@
|
|||
import { makeProviders, makeStandardFetcher, targets } from '../../lib/index.mjs';
|
||||
|
||||
(window as any).TEST = () => {
|
||||
makeProviders({
|
||||
fetcher: makeStandardFetcher(fetch),
|
||||
target: targets.ANY,
|
||||
});
|
||||
}
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"main": "startup.mjs"
|
||||
}
|
|
@ -0,0 +1,31 @@
|
|||
import { build, preview } from 'vite';
|
||||
import puppeteer from 'puppeteer';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { dirname } from 'path';
|
||||
|
||||
const root = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
await build({
|
||||
root,
|
||||
});
|
||||
const server = await preview({
|
||||
root,
|
||||
});
|
||||
let browser;
|
||||
try {
|
||||
browser = await puppeteer.launch({
|
||||
headless: 'new',
|
||||
args: ['--no-sandbox', '--disable-setuid-sandbox'],
|
||||
});
|
||||
const page = await browser.newPage();
|
||||
await page.goto(server.resolvedUrls.local[0]);
|
||||
await page.waitForFunction('!!window.TEST', { timeout: 5000 });
|
||||
await page.evaluate(() => {
|
||||
window.TEST();
|
||||
});
|
||||
} finally {
|
||||
server.httpServer.close();
|
||||
await browser.close();
|
||||
}
|
||||
|
||||
console.log('Success!');
|
|
@ -17,7 +17,7 @@
|
|||
}
|
||||
},
|
||||
"include": ["src"],
|
||||
"exclude": ["node_modules", "**/__tests__/*"],
|
||||
"exclude": ["node_modules", "**/__test__"],
|
||||
"ts-node": {
|
||||
"require": ["tsconfig-paths/register"]
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue