From 1dc7c879afd4d96e7513bf977bc55c2b8257f736 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Mon, 8 Jan 2024 16:27:59 +0100 Subject: [PATCH 01/22] add preferredHeaders to vidplay --- src/providers/embeds/vidplay/common.ts | 1 + src/providers/embeds/vidplay/index.ts | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/providers/embeds/vidplay/common.ts b/src/providers/embeds/vidplay/common.ts index 224e3dc..e3172f7 100644 --- a/src/providers/embeds/vidplay/common.ts +++ b/src/providers/embeds/vidplay/common.ts @@ -3,6 +3,7 @@ import { decodeData } from '@/providers/sources/vidsrcto/common'; import { EmbedScrapeContext } from '@/utils/context'; export const vidplayBase = 'https://vidplay.site'; +export const referer = 'https://vidplay.online/'; // This file is based on https://github.com/Ciarands/vidsrc-to-resolver/blob/dffa45e726a4b944cb9af0c9e7630476c93c0213/vidsrc.py#L16 // Full credits to @Ciarands! diff --git a/src/providers/embeds/vidplay/index.ts b/src/providers/embeds/vidplay/index.ts index 3c1f6a2..9d205e1 100644 --- a/src/providers/embeds/vidplay/index.ts +++ b/src/providers/embeds/vidplay/index.ts @@ -1,7 +1,8 @@ +import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; -import { getFileUrl } from './common'; +import { getFileUrl, referer } from './common'; import { SubtitleResult, VidplaySourceResponse } from './types'; export const vidplayScraper = makeEmbed({ @@ -46,6 +47,10 @@ export const vidplayScraper = makeEmbed({ playlist: source, flags: [], captions, + preferredHeaders: { + Referer: referer, + Origin: referer, + }, }, ], }; From 7697c767214921084e1ba6c1af3fd528e75f19a3 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Wed, 10 Jan 2024 21:21:47 +0100 Subject: [PATCH 02/22] add headers --- src/providers/embeds/upcloud.ts | 7 +++++++ src/providers/sources/remotestream.ts | 17 +++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/providers/embeds/upcloud.ts b/src/providers/embeds/upcloud.ts index 7880cc4..0f86cd0 100644 --- a/src/providers/embeds/upcloud.ts +++ b/src/providers/embeds/upcloud.ts @@ -4,6 +4,9 @@ import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; +const origin = 'https://rabbitstream.net'; +const referer = 'https://rabbitstream.net/'; + const { AES, enc } = crypto; interface StreamRes { @@ -126,6 +129,10 @@ export const upcloudScraper = makeEmbed({ playlist: sources.file, flags: [flags.CORS_ALLOWED], captions, + preferredHeaders: { + Referer: referer, + Origin: origin, + }, }, ], }; diff --git a/src/providers/sources/remotestream.ts b/src/providers/sources/remotestream.ts index 8a3090b..f0a0ae8 100644 --- a/src/providers/sources/remotestream.ts +++ b/src/providers/sources/remotestream.ts @@ -4,6 +4,9 @@ import { NotFoundError } from '@/utils/errors'; const remotestreamBase = atob('aHR0cHM6Ly9mc2IuOG1ldDNkdGpmcmNxY2hjb25xcGtsd3hzeGIyb2N1bWMuc3RyZWFt'); +const origin = 'https://remotestre.am'; +const referer = 'https://remotestre.am/'; + export const remotestreamScraper = makeSourcerer({ id: 'remotestream', name: 'Remote Stream', @@ -19,6 +22,9 @@ export const remotestreamScraper = makeSourcerer({ const streamRes = await ctx.fetcher.full(playlistLink, { method: 'HEAD', readHeaders: ['content-type'], + headers: { + Referer: referer, + }, }); if (!streamRes.headers.get('content-type')?.toLowerCase().includes('application/x-mpegurl')) throw new NotFoundError('No watchable item found'); @@ -33,6 +39,10 @@ export const remotestreamScraper = makeSourcerer({ playlist: playlistLink, type: 'hls', flags: [flags.CORS_ALLOWED], + preferredHeaders: { + Referer: referer, + Origin: origin, + }, }, ], }; @@ -44,6 +54,9 @@ export const remotestreamScraper = makeSourcerer({ const streamRes = await ctx.fetcher.full(playlistLink, { method: 'HEAD', readHeaders: ['content-type'], + headers: { + Referer: referer, + }, }); if (!streamRes.headers.get('content-type')?.toLowerCase().includes('application/x-mpegurl')) throw new NotFoundError('No watchable item found'); @@ -58,6 +71,10 @@ export const remotestreamScraper = makeSourcerer({ playlist: playlistLink, type: 'hls', flags: [flags.CORS_ALLOWED], + preferredHeaders: { + Referer: referer, + Origin: origin, + }, }, ], }; From 7a5cbb3428f325ac86dfdc270b56f0c018110de9 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Thu, 11 Jan 2024 19:14:21 +0100 Subject: [PATCH 03/22] Add CF_BLOCKED flags --- src/entrypoint/utils/targets.ts | 4 ++++ src/providers/sources/showbox/index.ts | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/entrypoint/utils/targets.ts b/src/entrypoint/utils/targets.ts index 16a02da..80988a4 100644 --- a/src/entrypoint/utils/targets.ts +++ b/src/entrypoint/utils/targets.ts @@ -5,6 +5,10 @@ export const flags = { // the stream is locked on IP, so only works if // request maker is same as player (not compatible with proxies) IP_LOCKED: 'ip-locked', + + // The source/embed is blocking cloudflare ip's + // This flag is not compatible with a proxy hosted on cloudflare + CF_BLOCKED: 'cf-blocked', } as const; export type Flags = (typeof flags)[keyof typeof flags]; diff --git a/src/providers/sources/showbox/index.ts b/src/providers/sources/showbox/index.ts index d6c4887..c8a834a 100644 --- a/src/providers/sources/showbox/index.ts +++ b/src/providers/sources/showbox/index.ts @@ -42,7 +42,7 @@ export const showboxScraper = makeSourcerer({ id: 'showbox', name: 'Showbox', rank: 300, - flags: [flags.CORS_ALLOWED], + flags: [flags.CORS_ALLOWED, flags.CF_BLOCKED], scrapeShow: comboScraper, scrapeMovie: comboScraper, }); From 9798659af817b5d036b16fa6214b24589a4478cc Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Jan 2024 17:18:12 +0100 Subject: [PATCH 04/22] add basic categories to tests --- package-lock.json | 26 ++++++-- package.json | 2 + src/__test__/providers/abc.test.ts | 7 ++ .../{ => standard}/fetchers/body.test.ts | 0 .../{ => standard}/fetchers/common.test.ts | 0 .../fetchers/simpleProxy.test.ts | 0 .../{ => standard}/fetchers/standard.test.ts | 0 src/__test__/{ => standard}/providerTests.ts | 66 +++++++++---------- .../{ => standard}/providers/checks.test.ts | 2 +- .../{ => standard}/runner/list.test.ts | 2 +- .../{ => standard}/runner/meta.test.ts | 2 +- .../{ => standard}/utils/features.test.ts | 0 .../{ => standard}/utils/list.test.ts | 0 .../{ => standard}/utils/valid.test.ts | 0 vite.config.js | 9 ++- 15 files changed, 71 insertions(+), 45 deletions(-) create mode 100644 src/__test__/providers/abc.test.ts rename src/__test__/{ => standard}/fetchers/body.test.ts (100%) rename src/__test__/{ => standard}/fetchers/common.test.ts (100%) rename src/__test__/{ => standard}/fetchers/simpleProxy.test.ts (100%) rename src/__test__/{ => standard}/fetchers/standard.test.ts (100%) rename src/__test__/{ => standard}/providerTests.ts (69%) rename src/__test__/{ => standard}/providers/checks.test.ts (98%) rename src/__test__/{ => standard}/runner/list.test.ts (98%) rename src/__test__/{ => standard}/runner/meta.test.ts (96%) rename src/__test__/{ => standard}/utils/features.test.ts (100%) rename src/__test__/{ => standard}/utils/list.test.ts (100%) rename src/__test__/{ => standard}/utils/valid.test.ts (100%) diff --git a/package-lock.json b/package-lock.json index f38bd59..9eebc03 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,6 +10,7 @@ "license": "MIT", "dependencies": { "cheerio": "^1.0.0-rc.12", + "cross-env": "^7.0.3", "crypto-js": "^4.1.1", "form-data": "^4.0.0", "iso-639-1": "^3.1.0", @@ -1794,6 +1795,23 @@ "dev": true, "license": "MIT" }, + "node_modules/cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "dependencies": { + "cross-spawn": "^7.0.1" + }, + "bin": { + "cross-env": "src/bin/cross-env.js", + "cross-env-shell": "src/bin/cross-env-shell.js" + }, + "engines": { + "node": ">=10.14", + "npm": ">=6", + "yarn": ">=1" + } + }, "node_modules/cross-fetch": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.0.0.tgz", @@ -1805,7 +1823,6 @@ }, "node_modules/cross-spawn": { "version": "7.0.3", - "dev": true, "license": "MIT", "dependencies": { "path-key": "^3.1.0", @@ -3585,8 +3602,7 @@ "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" }, "node_modules/iso-639-1": { "version": "3.1.0", @@ -4277,7 +4293,6 @@ }, "node_modules/path-key": { "version": "3.1.1", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -4785,7 +4800,6 @@ }, "node_modules/shebang-command": { "version": "2.0.0", - "dev": true, "license": "MIT", "dependencies": { "shebang-regex": "^3.0.0" @@ -4796,7 +4810,6 @@ }, "node_modules/shebang-regex": { "version": "3.0.0", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -5774,7 +5787,6 @@ }, "node_modules/which": { "version": "2.0.2", - "dev": true, "license": "ISC", "dependencies": { "isexe": "^2.0.0" diff --git a/package.json b/package.json index d4e7626..9bb36e9 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "cli": "ts-node ./src/dev-cli/index.ts", "test": "vitest run", "test:watch": "vitest", + "test:providers": "cross-env MW_TEST_PROVIDERS=true vitest run", "test:integration": "node ./tests/cjs && node ./tests/esm && node ./tests/browser", "test:coverage": "vitest run --coverage", "lint": "eslint --ext .ts,.js src/", @@ -78,6 +79,7 @@ }, "dependencies": { "cheerio": "^1.0.0-rc.12", + "cross-env": "^7.0.3", "crypto-js": "^4.1.1", "form-data": "^4.0.0", "iso-639-1": "^3.1.0", diff --git a/src/__test__/providers/abc.test.ts b/src/__test__/providers/abc.test.ts new file mode 100644 index 0000000..3aed2ea --- /dev/null +++ b/src/__test__/providers/abc.test.ts @@ -0,0 +1,7 @@ +import { describe, expect, it } from "vitest"; + +describe('abc', () => { + it('should do thing', () => { + expect(true).toBe(false); + }) +}) diff --git a/src/__test__/fetchers/body.test.ts b/src/__test__/standard/fetchers/body.test.ts similarity index 100% rename from src/__test__/fetchers/body.test.ts rename to src/__test__/standard/fetchers/body.test.ts diff --git a/src/__test__/fetchers/common.test.ts b/src/__test__/standard/fetchers/common.test.ts similarity index 100% rename from src/__test__/fetchers/common.test.ts rename to src/__test__/standard/fetchers/common.test.ts diff --git a/src/__test__/fetchers/simpleProxy.test.ts b/src/__test__/standard/fetchers/simpleProxy.test.ts similarity index 100% rename from src/__test__/fetchers/simpleProxy.test.ts rename to src/__test__/standard/fetchers/simpleProxy.test.ts diff --git a/src/__test__/fetchers/standard.test.ts b/src/__test__/standard/fetchers/standard.test.ts similarity index 100% rename from src/__test__/fetchers/standard.test.ts rename to src/__test__/standard/fetchers/standard.test.ts diff --git a/src/__test__/providerTests.ts b/src/__test__/standard/providerTests.ts similarity index 69% rename from src/__test__/providerTests.ts rename to src/__test__/standard/providerTests.ts index 551a3ec..f5c87b7 100644 --- a/src/__test__/providerTests.ts +++ b/src/__test__/standard/providerTests.ts @@ -2,7 +2,7 @@ import { vi } from 'vitest'; import { gatherAllEmbeds, gatherAllSources } from '@/providers/all'; -import { Embed, Sourcerer } from '@/providers/base'; +import { makeEmbed, makeSourcerer } from '@/providers/base'; export function makeProviderMocks() { const embedsMock = vi.fn, ReturnType>(); @@ -13,104 +13,104 @@ export function makeProviderMocks() { }; } -const sourceA = { +const sourceA = makeSourcerer({ id: 'a', name: 'A', rank: 1, disabled: false, flags: [], -} as Sourcerer; -const sourceB = { +}); +const sourceB = makeSourcerer({ id: 'b', name: 'B', rank: 2, disabled: false, flags: [], -} as Sourcerer; -const sourceCDisabled = { +}); +const sourceCDisabled = makeSourcerer({ id: 'c', name: 'C', rank: 3, disabled: true, flags: [], -} as Sourcerer; -const sourceAHigherRank = { +}); +const sourceAHigherRank = makeSourcerer({ id: 'a', name: 'A', rank: 100, disabled: false, flags: [], -} as Sourcerer; -const sourceGSameRankAsA = { +}); +const sourceGSameRankAsA = makeSourcerer({ id: 'g', name: 'G', rank: 1, disabled: false, flags: [], -} as Sourcerer; -const fullSourceYMovie = { +}); +const fullSourceYMovie = makeSourcerer({ id: 'y', name: 'Y', rank: 105, scrapeMovie: vi.fn(), flags: [], -} as Sourcerer; -const fullSourceYShow = { +}); +const fullSourceYShow = makeSourcerer({ id: 'y', name: 'Y', rank: 105, scrapeShow: vi.fn(), flags: [], -} as Sourcerer; -const fullSourceZBoth = { +}); +const fullSourceZBoth = makeSourcerer({ id: 'z', name: 'Z', rank: 106, scrapeMovie: vi.fn(), scrapeShow: vi.fn(), flags: [], -} as Sourcerer; +}); -const embedD = { +const embedD = makeEmbed({ id: 'd', rank: 4, disabled: false, -} as Embed; -const embedA = { +} as any); +const embedA = makeEmbed({ id: 'a', rank: 5, disabled: false, -} as Embed; -const embedEDisabled = { +} as any); +const embedEDisabled = makeEmbed({ id: 'e', rank: 6, disabled: true, -} as Embed; -const embedDHigherRank = { +} as any); +const embedDHigherRank = makeEmbed({ id: 'd', rank: 4000, disabled: false, -} as Embed; -const embedFSameRankAsA = { +} as any); +const embedFSameRankAsA = makeEmbed({ id: 'f', rank: 5, disabled: false, -} as Embed; -const embedHSameRankAsSourceA = { +} as any); +const embedHSameRankAsSourceA = makeEmbed({ id: 'h', rank: 1, disabled: false, -} as Embed; -const fullEmbedX = { +} as any); +const fullEmbedX = makeEmbed({ id: 'x', name: 'X', rank: 104, -} as Embed; -const fullEmbedZ = { +} as any); +const fullEmbedZ = makeEmbed({ id: 'z', name: 'Z', rank: 109, -} as Embed; +} as any); export const mockSources = { sourceA, diff --git a/src/__test__/providers/checks.test.ts b/src/__test__/standard/providers/checks.test.ts similarity index 98% rename from src/__test__/providers/checks.test.ts rename to src/__test__/standard/providers/checks.test.ts index 404fb31..747c240 100644 --- a/src/__test__/providers/checks.test.ts +++ b/src/__test__/standard/providers/checks.test.ts @@ -1,4 +1,4 @@ -import { mockEmbeds, mockSources } from '@/__test__/providerTests'; +import { mockEmbeds, mockSources } from '../providerTests'; import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers'; import { FeatureMap } from '@/entrypoint/utils/targets'; import { getProviders } from '@/providers/get'; diff --git a/src/__test__/runner/list.test.ts b/src/__test__/standard/runner/list.test.ts similarity index 98% rename from src/__test__/runner/list.test.ts rename to src/__test__/standard/runner/list.test.ts index 336de44..b615457 100644 --- a/src/__test__/runner/list.test.ts +++ b/src/__test__/standard/runner/list.test.ts @@ -1,4 +1,4 @@ -import { mockEmbeds, mockSources } from '@/__test__/providerTests'; +import { mockEmbeds, mockSources } from '../providerTests.ts'; import { makeProviders } from '@/entrypoint/declare'; import { targets } from '@/entrypoint/utils/targets'; import { afterEach, describe, expect, it, vi } from 'vitest'; diff --git a/src/__test__/runner/meta.test.ts b/src/__test__/standard/runner/meta.test.ts similarity index 96% rename from src/__test__/runner/meta.test.ts rename to src/__test__/standard/runner/meta.test.ts index 423a8e6..ac2fab6 100644 --- a/src/__test__/runner/meta.test.ts +++ b/src/__test__/standard/runner/meta.test.ts @@ -1,4 +1,4 @@ -import { mockEmbeds, mockSources } from '@/__test__/providerTests'; +import { mockEmbeds, mockSources } from '../providerTests.ts'; import { makeProviders } from '@/entrypoint/declare'; import { targets } from '@/entrypoint/utils/targets'; import { afterEach, describe, expect, it, vi } from 'vitest'; diff --git a/src/__test__/utils/features.test.ts b/src/__test__/standard/utils/features.test.ts similarity index 100% rename from src/__test__/utils/features.test.ts rename to src/__test__/standard/utils/features.test.ts diff --git a/src/__test__/utils/list.test.ts b/src/__test__/standard/utils/list.test.ts similarity index 100% rename from src/__test__/utils/list.test.ts rename to src/__test__/standard/utils/list.test.ts diff --git a/src/__test__/utils/valid.test.ts b/src/__test__/standard/utils/valid.test.ts similarity index 100% rename from src/__test__/utils/valid.test.ts rename to src/__test__/standard/utils/valid.test.ts diff --git a/vite.config.js b/vite.config.js index e5d4f10..8f6597d 100644 --- a/vite.config.js +++ b/vite.config.js @@ -5,7 +5,9 @@ const dts = require('vite-plugin-dts'); const pkg = require('./package.json'); const fs = require('fs/promises'); -const main = path.resolve(__dirname, 'src/index.ts'); +const shouldTestProviders = process.env.MW_TEST_PROVIDERS === "true" +let tests = ['src/__test__/standard/**/*.test.ts']; +if (shouldTestProviders) tests = ['src/__test__/providers/**/*.test.ts'] module.exports = defineConfig({ plugins: [ @@ -34,10 +36,13 @@ module.exports = defineConfig({ outDir: 'lib', lib: { - entry: main, + entry: path.resolve(__dirname, 'src/index.ts'), name: 'index', fileName: 'index', formats: ['umd', 'es'], }, }, + test: { + include: tests + } }); From 528e2774b530fe095d6717837e912ba9d1cf224b Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Jan 2024 18:03:10 +0100 Subject: [PATCH 05/22] Add basic provider unit testing --- src/__test__/providers/abc.test.ts | 7 -- src/__test__/providers/providerUtils.ts | 88 ++++++++++++++++++++++++ src/__test__/providers/providers.test.ts | 23 +++++++ src/__test__/providers/testMedia.ts | 30 ++++++++ 4 files changed, 141 insertions(+), 7 deletions(-) delete mode 100644 src/__test__/providers/abc.test.ts create mode 100644 src/__test__/providers/providerUtils.ts create mode 100644 src/__test__/providers/providers.test.ts create mode 100644 src/__test__/providers/testMedia.ts diff --git a/src/__test__/providers/abc.test.ts b/src/__test__/providers/abc.test.ts deleted file mode 100644 index 3aed2ea..0000000 --- a/src/__test__/providers/abc.test.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { describe, expect, it } from "vitest"; - -describe('abc', () => { - it('should do thing', () => { - expect(true).toBe(false); - }) -}) diff --git a/src/__test__/providers/providerUtils.ts b/src/__test__/providers/providerUtils.ts new file mode 100644 index 0000000..b9b2bc8 --- /dev/null +++ b/src/__test__/providers/providerUtils.ts @@ -0,0 +1,88 @@ +import { ScrapeMedia } from "@/entrypoint/utils/media"; +import { Sourcerer } from "@/providers/base"; +import { buildProviders } from "@/entrypoint/builder"; +import { describe, expect, it } from "vitest"; +import { makeStandardFetcher } from "@/fetchers/standardFetch"; +import { ProviderControls } from "@/entrypoint/controls"; +import { NotFoundError } from "@/utils/errors"; +import { targets } from "@/entrypoint/utils/targets"; +import { getAllEmbedMetaSorted } from "@/entrypoint/utils/meta"; +import { getBuiltinEmbeds } from "@/entrypoint/providers"; + +export type TestTypes = 'standard' | 'ip:standard'; + +export interface TestSourceOptions { + source: Sourcerer; + testSuite: ScrapeMedia[]; + types: TestTypes[]; + debug?: boolean; + expect: { + embeds?: number; + streams?: number; + error?: boolean; + notfound?: boolean; + } +} + +// TODO add proxy support + +function makeBaseProviders() { + const builder = buildProviders() + .setTarget(targets.ANY) + .setFetcher(makeStandardFetcher(fetch)); + const embeds = getBuiltinEmbeds(); + embeds.forEach(embed => builder.addEmbed(embed)); + return builder; +} + +export function testSource(ops: TestSourceOptions) { + if (ops.testSuite.length === 0) throw new Error("Test suite must have at least one test"); + describe(`source:${ops.source.id}`, () => { + ops.testSuite.forEach((test, i) => { + async function runTest(providers: ProviderControls) { + let hasNotFound = false; + let hasError = false; + let streamCount = 0; + let embedCount = 0; + try { + const result = await providers.runSourceScraper({ + id: ops.source.id, + media: test, + }) + if (ops.debug) console.log(result); + streamCount = (result.stream ?? []).length; + embedCount = result.embeds.length; + } catch (err) { + if (ops.debug) console.log(err); + if (err instanceof NotFoundError) + hasNotFound = true; + else + hasError = true; + } + expect(ops.expect.error ?? false).toBe(hasError); + expect(ops.expect.notfound ?? false).toBe(hasNotFound); + expect(ops.expect.streams ?? 0).toBe(streamCount); + expect(ops.expect.embeds ?? 0).toBe(embedCount); + } + + if (ops.types.includes('standard')) { + it(`Should pass test ${i} - standard`, async () => { + const providers = makeBaseProviders() + .addSource(ops.source) + .build(); + await runTest(providers); + }) + } + + if (ops.types.includes('ip:standard')) { + it(`Should pass test ${i} - standard:ip`, async () => { + const providers = makeBaseProviders() + .addSource(ops.source) + .enableConsistentIpForRequests() + .build(); + await runTest(providers); + }) + } + }) + }) +} diff --git a/src/__test__/providers/providers.test.ts b/src/__test__/providers/providers.test.ts new file mode 100644 index 0000000..8a2165e --- /dev/null +++ b/src/__test__/providers/providers.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; +import { testSource } from "./providerUtils"; +import { lookmovieScraper } from "@/providers/sources/lookmovie"; +import { testMedia } from "./testMedia"; +import { showboxScraper } from "@/providers/sources/showbox"; + +testSource({ + source: lookmovieScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['ip:standard'], + expect: { + streams: 1, + } +}) + +testSource({ + source: showboxScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard'], + expect: { + embeds: 1, + } +}) diff --git a/src/__test__/providers/testMedia.ts b/src/__test__/providers/testMedia.ts new file mode 100644 index 0000000..0e0d050 --- /dev/null +++ b/src/__test__/providers/testMedia.ts @@ -0,0 +1,30 @@ +import { ScrapeMedia } from "@/entrypoint/utils/media"; + +function makeMedia(media: ScrapeMedia): ScrapeMedia { + return media; +} + +export const testMedia = { + arcane: makeMedia({ + type: "show", + title: "Arcane", + tmdbId: "94605", + releaseYear: 2021, + episode: { + number: 1, + tmdbId: '1953812', + }, + season: { + number: 1, + tmdbId: '134187', + }, + imdbId: 'tt11126994' + }), + hamilton: makeMedia({ + type: 'movie', + tmdbId: '556574', + imdbId: 'tt8503618', + releaseYear: 2020, + title: 'Hamilton' + }) +} From 327ba025c2000625af7a474de682640473383dac Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Jan 2024 18:18:08 +0100 Subject: [PATCH 06/22] Enabled febbox subtitles --- src/__test__/providers/providers.test.ts | 3 ++- src/providers/embeds/febbox/mp4.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/__test__/providers/providers.test.ts b/src/__test__/providers/providers.test.ts index 8a2165e..6114f37 100644 --- a/src/__test__/providers/providers.test.ts +++ b/src/__test__/providers/providers.test.ts @@ -1,9 +1,10 @@ -import { describe, expect, it } from "vitest"; import { testSource } from "./providerUtils"; import { lookmovieScraper } from "@/providers/sources/lookmovie"; import { testMedia } from "./testMedia"; import { showboxScraper } from "@/providers/sources/showbox"; +// TODO add embed testing + testSource({ source: lookmovieScraper, testSuite: [testMedia.arcane, testMedia.hamilton], diff --git a/src/providers/embeds/febbox/mp4.ts b/src/providers/embeds/febbox/mp4.ts index b69812f..1122e53 100644 --- a/src/providers/embeds/febbox/mp4.ts +++ b/src/providers/embeds/febbox/mp4.ts @@ -2,6 +2,7 @@ import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { parseInputUrl } from '@/providers/embeds/febbox/common'; import { getStreamQualities } from '@/providers/embeds/febbox/qualities'; +import { getSubtitles } from '@/providers/embeds/febbox/subtitles'; export const febboxMp4Scraper = makeEmbed({ id: 'febbox-mp4', @@ -41,7 +42,7 @@ export const febboxMp4Scraper = makeEmbed({ stream: [ { id: 'primary', - captions: [], // subtitles temporarily disabled, the endpoints are broken + captions: await getSubtitles(ctx, id, fid, type, episode, season), qualities, type: 'file', flags: [flags.CORS_ALLOWED], From 616c8a1e1b9a6a0bb7d8d7ce209e3ef3ff1e022f Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Jan 2024 18:34:32 +0100 Subject: [PATCH 07/22] Add embed testing --- src/__test__/providers/embeds.test.ts | 17 +++++ src/__test__/providers/providerUtils.ts | 94 ++++++++++++++++++++++-- src/__test__/providers/providers.test.ts | 5 +- 3 files changed, 108 insertions(+), 8 deletions(-) create mode 100644 src/__test__/providers/embeds.test.ts diff --git a/src/__test__/providers/embeds.test.ts b/src/__test__/providers/embeds.test.ts new file mode 100644 index 0000000..5775a91 --- /dev/null +++ b/src/__test__/providers/embeds.test.ts @@ -0,0 +1,17 @@ +import { febboxMp4Scraper } from "@/providers/embeds/febbox/mp4"; +import { testEmbed } from "./providerUtils"; +import dotenv from 'dotenv'; + +dotenv.config(); + +testEmbed({ + embed: febboxMp4Scraper, + testUrls: [ + '/show/16448/1/1', + '/movie/27769//' + ], + types: ['standard', 'proxied'], + expect: { + streams: 1, + } +}) diff --git a/src/__test__/providers/providerUtils.ts b/src/__test__/providers/providerUtils.ts index b9b2bc8..4675f86 100644 --- a/src/__test__/providers/providerUtils.ts +++ b/src/__test__/providers/providerUtils.ts @@ -1,15 +1,15 @@ import { ScrapeMedia } from "@/entrypoint/utils/media"; -import { Sourcerer } from "@/providers/base"; +import { Embed, Sourcerer } from "@/providers/base"; import { buildProviders } from "@/entrypoint/builder"; import { describe, expect, it } from "vitest"; import { makeStandardFetcher } from "@/fetchers/standardFetch"; import { ProviderControls } from "@/entrypoint/controls"; import { NotFoundError } from "@/utils/errors"; import { targets } from "@/entrypoint/utils/targets"; -import { getAllEmbedMetaSorted } from "@/entrypoint/utils/meta"; import { getBuiltinEmbeds } from "@/entrypoint/providers"; +import { makeSimpleProxyFetcher } from "@/fetchers/simpleProxy"; -export type TestTypes = 'standard' | 'ip:standard'; +export type TestTypes = 'standard' | 'ip:standard' | 'proxied'; export interface TestSourceOptions { source: Sourcerer; @@ -24,14 +24,28 @@ export interface TestSourceOptions { } } -// TODO add proxy support +export interface TestEmbedOptions { + embed: Embed; + testUrls: string[]; + types: TestTypes[]; + debug?: boolean; + expect: { + streams?: number; + error?: boolean; + } +} function makeBaseProviders() { + const builder = makeBaseEmbedProviders(); + const embeds = getBuiltinEmbeds(); + embeds.forEach(embed => builder.addEmbed(embed)); + return builder; +} + +function makeBaseEmbedProviders() { const builder = buildProviders() .setTarget(targets.ANY) .setFetcher(makeStandardFetcher(fetch)); - const embeds = getBuiltinEmbeds(); - embeds.forEach(embed => builder.addEmbed(embed)); return builder; } @@ -83,6 +97,74 @@ export function testSource(ops: TestSourceOptions) { await runTest(providers); }) } + + if (ops.types.includes('proxied')) { + it(`Should pass test ${i} - proxied`, async () => { + if (!process.env.MOVIE_WEB_PROXY_URL) + throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); + const providers = makeBaseProviders() + .addSource(ops.source) + .setProxiedFetcher(makeSimpleProxyFetcher(process.env.MOVIE_WEB_PROXY_URL, fetch)) + .build(); + await runTest(providers); + }) + } + }) + }) +} + +export function testEmbed(ops: TestEmbedOptions) { + if (ops.testUrls.length === 0) throw new Error("Test urls must have at least one url"); + describe(`embed:${ops.embed.id}`, () => { + ops.testUrls.forEach((test, i) => { + async function runTest(providers: ProviderControls) { + let hasError = false; + let streamCount = 0; + try { + const result = await providers.runEmbedScraper({ + id: ops.embed.id, + url: test, + }) + if (ops.debug) console.log(result); + streamCount = (result.stream ?? []).length; + } catch (err) { + if (ops.debug) console.log(err); + hasError = true; + } + expect(ops.expect.error ?? false).toBe(hasError); + expect(ops.expect.streams ?? 0).toBe(streamCount); + } + + if (ops.types.includes('standard')) { + it(`Should pass test ${i} - standard`, async () => { + const providers = makeBaseEmbedProviders() + .addEmbed(ops.embed) + .build(); + await runTest(providers); + }) + } + + if (ops.types.includes('ip:standard')) { + it(`Should pass test ${i} - standard:ip`, async () => { + const providers = makeBaseEmbedProviders() + .addEmbed(ops.embed) + .enableConsistentIpForRequests() + .build(); + await runTest(providers); + }) + } + + if (ops.types.includes('proxied')) { + it(`Should pass test ${i} - proxied`, async () => { + if (!process.env.MOVIE_WEB_PROXY_URL) + throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); + const providers = makeBaseEmbedProviders() + .addEmbed(ops.embed) + .setProxiedFetcher(makeSimpleProxyFetcher(process.env.MOVIE_WEB_PROXY_URL, fetch)) + .build(); + await runTest(providers); + }) + } }) }) } diff --git a/src/__test__/providers/providers.test.ts b/src/__test__/providers/providers.test.ts index 6114f37..0f29dc1 100644 --- a/src/__test__/providers/providers.test.ts +++ b/src/__test__/providers/providers.test.ts @@ -2,8 +2,9 @@ import { testSource } from "./providerUtils"; import { lookmovieScraper } from "@/providers/sources/lookmovie"; import { testMedia } from "./testMedia"; import { showboxScraper } from "@/providers/sources/showbox"; +import dotenv from 'dotenv'; -// TODO add embed testing +dotenv.config(); testSource({ source: lookmovieScraper, @@ -17,7 +18,7 @@ testSource({ testSource({ source: showboxScraper, testSuite: [testMedia.arcane, testMedia.hamilton], - types: ['standard'], + types: ['standard', 'proxied'], expect: { embeds: 1, } From 03a628ea34de115f4f9e3a2ac9a7601b8456e836 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Jan 2024 18:54:42 +0100 Subject: [PATCH 08/22] Improve logging of provider unit tests --- package.json | 2 +- src/__test__/providers/providerUtils.ts | 197 ++++++++++++------------ 2 files changed, 102 insertions(+), 97 deletions(-) diff --git a/package.json b/package.json index 9bb36e9..d8a8680 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "cli": "ts-node ./src/dev-cli/index.ts", "test": "vitest run", "test:watch": "vitest", - "test:providers": "cross-env MW_TEST_PROVIDERS=true vitest run", + "test:providers": "cross-env MW_TEST_PROVIDERS=true vitest run --reporter verbose", "test:integration": "node ./tests/cjs && node ./tests/esm && node ./tests/browser", "test:coverage": "vitest run --coverage", "lint": "eslint --ext .ts,.js src/", diff --git a/src/__test__/providers/providerUtils.ts b/src/__test__/providers/providerUtils.ts index 4675f86..f50d53f 100644 --- a/src/__test__/providers/providerUtils.ts +++ b/src/__test__/providers/providerUtils.ts @@ -53,62 +53,65 @@ export function testSource(ops: TestSourceOptions) { if (ops.testSuite.length === 0) throw new Error("Test suite must have at least one test"); describe(`source:${ops.source.id}`, () => { ops.testSuite.forEach((test, i) => { - async function runTest(providers: ProviderControls) { - let hasNotFound = false; - let hasError = false; - let streamCount = 0; - let embedCount = 0; - try { - const result = await providers.runSourceScraper({ - id: ops.source.id, - media: test, - }) - if (ops.debug) console.log(result); - streamCount = (result.stream ?? []).length; - embedCount = result.embeds.length; - } catch (err) { - if (ops.debug) console.log(err); - if (err instanceof NotFoundError) - hasNotFound = true; - else - hasError = true; + describe(`test ${i}`, () => { + async function runTest(providers: ProviderControls) { + let hasNotFound = false; + let hasError = false; + let streamCount = 0; + let embedCount = 0; + try { + const result = await providers.runSourceScraper({ + id: ops.source.id, + media: test, + }) + if (ops.debug) console.log(result); + streamCount = (result.stream ?? []).length; + embedCount = result.embeds.length; + } catch (err) { + if (ops.debug) console.log(err); + if (err instanceof NotFoundError) + hasNotFound = true; + else + hasError = true; + } + expect(ops.expect.error ?? false).toBe(hasError); + expect(ops.expect.notfound ?? false).toBe(hasNotFound); + expect(ops.expect.streams ?? 0).toBe(streamCount); + expect(ops.expect.embeds ?? 0).toBe(embedCount); } - expect(ops.expect.error ?? false).toBe(hasError); - expect(ops.expect.notfound ?? false).toBe(hasNotFound); - expect(ops.expect.streams ?? 0).toBe(streamCount); - expect(ops.expect.embeds ?? 0).toBe(embedCount); - } - if (ops.types.includes('standard')) { - it(`Should pass test ${i} - standard`, async () => { - const providers = makeBaseProviders() - .addSource(ops.source) - .build(); - await runTest(providers); - }) - } + if (ops.types.includes('standard')) { + it(`Should pass test ${i} - standard`, async () => { + const providers = makeBaseProviders() + .addSource(ops.source) + .build(); + await runTest(providers); + }) + } - if (ops.types.includes('ip:standard')) { - it(`Should pass test ${i} - standard:ip`, async () => { - const providers = makeBaseProviders() - .addSource(ops.source) - .enableConsistentIpForRequests() - .build(); - await runTest(providers); - }) - } + if (ops.types.includes('ip:standard')) { + it(`Should pass test ${i} - standard:ip`, async () => { + const providers = makeBaseProviders() + .addSource(ops.source) + .enableConsistentIpForRequests() + .build(); + await runTest(providers); + }) + } - if (ops.types.includes('proxied')) { - it(`Should pass test ${i} - proxied`, async () => { - if (!process.env.MOVIE_WEB_PROXY_URL) - throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); - const providers = makeBaseProviders() - .addSource(ops.source) - .setProxiedFetcher(makeSimpleProxyFetcher(process.env.MOVIE_WEB_PROXY_URL, fetch)) - .build(); - await runTest(providers); - }) - } + if (ops.types.includes('proxied')) { + it(`Should pass test ${i} - proxied`, async () => { + if (!process.env.MOVIE_WEB_PROXY_URL) + throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); + const providers = makeBaseProviders() + .addSource(ops.source) + .setProxiedFetcher(makeSimpleProxyFetcher(process.env.MOVIE_WEB_PROXY_URL, fetch)) + .build(); + await runTest(providers); + }) + } + + }) }) }) } @@ -117,54 +120,56 @@ export function testEmbed(ops: TestEmbedOptions) { if (ops.testUrls.length === 0) throw new Error("Test urls must have at least one url"); describe(`embed:${ops.embed.id}`, () => { ops.testUrls.forEach((test, i) => { - async function runTest(providers: ProviderControls) { - let hasError = false; - let streamCount = 0; - try { - const result = await providers.runEmbedScraper({ - id: ops.embed.id, - url: test, - }) - if (ops.debug) console.log(result); - streamCount = (result.stream ?? []).length; - } catch (err) { - if (ops.debug) console.log(err); - hasError = true; + describe(`test ${i}`, () => { + async function runTest(providers: ProviderControls) { + let hasError = false; + let streamCount = 0; + try { + const result = await providers.runEmbedScraper({ + id: ops.embed.id, + url: test, + }) + if (ops.debug) console.log(result); + streamCount = (result.stream ?? []).length; + } catch (err) { + if (ops.debug) console.log(err); + hasError = true; + } + expect(ops.expect.error ?? false).toBe(hasError); + expect(ops.expect.streams ?? 0).toBe(streamCount); } - expect(ops.expect.error ?? false).toBe(hasError); - expect(ops.expect.streams ?? 0).toBe(streamCount); - } - if (ops.types.includes('standard')) { - it(`Should pass test ${i} - standard`, async () => { - const providers = makeBaseEmbedProviders() - .addEmbed(ops.embed) - .build(); - await runTest(providers); - }) - } + if (ops.types.includes('standard')) { + it(`Should pass test ${i} - standard`, async () => { + const providers = makeBaseEmbedProviders() + .addEmbed(ops.embed) + .build(); + await runTest(providers); + }) + } - if (ops.types.includes('ip:standard')) { - it(`Should pass test ${i} - standard:ip`, async () => { - const providers = makeBaseEmbedProviders() - .addEmbed(ops.embed) - .enableConsistentIpForRequests() - .build(); - await runTest(providers); - }) - } + if (ops.types.includes('ip:standard')) { + it(`Should pass test ${i} - standard:ip`, async () => { + const providers = makeBaseEmbedProviders() + .addEmbed(ops.embed) + .enableConsistentIpForRequests() + .build(); + await runTest(providers); + }) + } - if (ops.types.includes('proxied')) { - it(`Should pass test ${i} - proxied`, async () => { - if (!process.env.MOVIE_WEB_PROXY_URL) - throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); - const providers = makeBaseEmbedProviders() - .addEmbed(ops.embed) - .setProxiedFetcher(makeSimpleProxyFetcher(process.env.MOVIE_WEB_PROXY_URL, fetch)) - .build(); - await runTest(providers); - }) - } + if (ops.types.includes('proxied')) { + it(`Should pass test ${i} - proxied`, async () => { + if (!process.env.MOVIE_WEB_PROXY_URL) + throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); + const providers = makeBaseEmbedProviders() + .addEmbed(ops.embed) + .setProxiedFetcher(makeSimpleProxyFetcher(process.env.MOVIE_WEB_PROXY_URL, fetch)) + .build(); + await runTest(providers); + }) + } + }); }) }) } From 949fe487bc09faca006411963268f6d2eae4c690 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Jan 2024 19:22:37 +0100 Subject: [PATCH 09/22] fix some stuff --- package-lock.json | 11 +++++++++-- package.json | 2 +- src/__test__/providers/providerUtils.ts | 12 ++++++------ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9eebc03..9921a7b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,6 @@ "license": "MIT", "dependencies": { "cheerio": "^1.0.0-rc.12", - "cross-env": "^7.0.3", "crypto-js": "^4.1.1", "form-data": "^4.0.0", "iso-639-1": "^3.1.0", @@ -27,6 +26,7 @@ "@typescript-eslint/parser": "^5.60.0", "@vitest/coverage-v8": "^0.34.3", "commander": "^11.0.0", + "cross-env": "^7.0.3", "dotenv": "^16.3.1", "enquirer": "^2.4.1", "eslint": "^8.30.0", @@ -1799,6 +1799,7 @@ "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "dev": true, "dependencies": { "cross-spawn": "^7.0.1" }, @@ -1823,6 +1824,7 @@ }, "node_modules/cross-spawn": { "version": "7.0.3", + "dev": true, "license": "MIT", "dependencies": { "path-key": "^3.1.0", @@ -3602,7 +3604,8 @@ "node_modules/isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true }, "node_modules/iso-639-1": { "version": "3.1.0", @@ -4293,6 +4296,7 @@ }, "node_modules/path-key": { "version": "3.1.1", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -4800,6 +4804,7 @@ }, "node_modules/shebang-command": { "version": "2.0.0", + "dev": true, "license": "MIT", "dependencies": { "shebang-regex": "^3.0.0" @@ -4810,6 +4815,7 @@ }, "node_modules/shebang-regex": { "version": "3.0.0", + "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -5787,6 +5793,7 @@ }, "node_modules/which": { "version": "2.0.2", + "dev": true, "license": "ISC", "dependencies": { "isexe": "^2.0.0" diff --git a/package.json b/package.json index d8a8680..4047136 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "@typescript-eslint/parser": "^5.60.0", "@vitest/coverage-v8": "^0.34.3", "commander": "^11.0.0", + "cross-env": "^7.0.3", "dotenv": "^16.3.1", "enquirer": "^2.4.1", "eslint": "^8.30.0", @@ -79,7 +80,6 @@ }, "dependencies": { "cheerio": "^1.0.0-rc.12", - "cross-env": "^7.0.3", "crypto-js": "^4.1.1", "form-data": "^4.0.0", "iso-639-1": "^3.1.0", diff --git a/src/__test__/providers/providerUtils.ts b/src/__test__/providers/providerUtils.ts index f50d53f..65fe710 100644 --- a/src/__test__/providers/providerUtils.ts +++ b/src/__test__/providers/providerUtils.ts @@ -81,7 +81,7 @@ export function testSource(ops: TestSourceOptions) { } if (ops.types.includes('standard')) { - it(`Should pass test ${i} - standard`, async () => { + it(`standard`, async () => { const providers = makeBaseProviders() .addSource(ops.source) .build(); @@ -90,7 +90,7 @@ export function testSource(ops: TestSourceOptions) { } if (ops.types.includes('ip:standard')) { - it(`Should pass test ${i} - standard:ip`, async () => { + it(`standard:ip`, async () => { const providers = makeBaseProviders() .addSource(ops.source) .enableConsistentIpForRequests() @@ -100,7 +100,7 @@ export function testSource(ops: TestSourceOptions) { } if (ops.types.includes('proxied')) { - it(`Should pass test ${i} - proxied`, async () => { + it(`proxied`, async () => { if (!process.env.MOVIE_WEB_PROXY_URL) throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); const providers = makeBaseProviders() @@ -140,7 +140,7 @@ export function testEmbed(ops: TestEmbedOptions) { } if (ops.types.includes('standard')) { - it(`Should pass test ${i} - standard`, async () => { + it(`standard`, async () => { const providers = makeBaseEmbedProviders() .addEmbed(ops.embed) .build(); @@ -149,7 +149,7 @@ export function testEmbed(ops: TestEmbedOptions) { } if (ops.types.includes('ip:standard')) { - it(`Should pass test ${i} - standard:ip`, async () => { + it(`standard:ip`, async () => { const providers = makeBaseEmbedProviders() .addEmbed(ops.embed) .enableConsistentIpForRequests() @@ -159,7 +159,7 @@ export function testEmbed(ops: TestEmbedOptions) { } if (ops.types.includes('proxied')) { - it(`Should pass test ${i} - proxied`, async () => { + it(`proxied`, async () => { if (!process.env.MOVIE_WEB_PROXY_URL) throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); const providers = makeBaseEmbedProviders() From e8a8909ecb4cce8bb3484ce6058d6e019e5125f7 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 17 Jan 2024 21:15:56 +0100 Subject: [PATCH 10/22] Add better embed testing + add everything --- src/__test__/providers/embedUtils.ts | 90 ++++++++++++++++++ src/__test__/providers/embeds.test.ts | 113 +++++++++++++++++++++-- src/__test__/providers/providerUtils.ts | 85 ++--------------- src/__test__/providers/providers.test.ts | 70 ++++++++++++++ 4 files changed, 273 insertions(+), 85 deletions(-) create mode 100644 src/__test__/providers/embedUtils.ts diff --git a/src/__test__/providers/embedUtils.ts b/src/__test__/providers/embedUtils.ts new file mode 100644 index 0000000..3ba88a3 --- /dev/null +++ b/src/__test__/providers/embedUtils.ts @@ -0,0 +1,90 @@ +import { buildProviders } from "@/entrypoint/builder"; +import { ScrapeMedia } from "@/entrypoint/utils/media"; +import { targets } from "@/entrypoint/utils/targets"; +import { makeStandardFetcher } from "@/fetchers/standardFetch"; +import { Embed, Sourcerer, SourcererEmbed } from "@/providers/base"; +import { TestTypes } from "./providerUtils"; +import { describe, expect, it } from "vitest"; +import { ProviderControls } from "@/entrypoint/controls"; +import { makeSimpleProxyFetcher } from "@/fetchers/simpleProxy"; + +export interface TestEmbedOptions { + embed: Embed; + source: Sourcerer; + testSuite: ScrapeMedia[]; + types: TestTypes[]; + debug?: boolean; + expect: { + embeds: number; + streams?: number; + error?: boolean; + } +} + +function makeBaseEmbedProviders() { + const builder = buildProviders() + .setTarget(targets.ANY) + .setFetcher(makeStandardFetcher(fetch)); + return builder; +} + +export function testEmbed(ops: TestEmbedOptions) { + if (ops.testSuite.length === 0) throw new Error("Test suite must have at least one test"); + describe(`embed:${ops.source.id}:${ops.embed.id}`, () => { + ops.testSuite.forEach((test) => { + describe(`test ${test.title}`, async () => { + async function gatherEmbeds(providers: ProviderControls): Promise { + const results = await providers.runSourceScraper({ + id: ops.source.id, + media: test, + }) + if (results.embeds.length !== ops.expect.embeds) throw new Error(`Embeds don't match expected amount of embeds (${ops.source.id}, ${ops.embed.id}, got ${results.embeds.length} but expected ${ops.expect.embeds})`); + return results.embeds; + } + + async function runTest(providers: ProviderControls, embedUrl: string) { + let hasError = false; + let streamCount = 0; + try { + const result = await providers.runEmbedScraper({ + id: ops.embed.id, + url: embedUrl, + }) + if (ops.debug) console.log(result); + streamCount = (result.stream ?? []).length; + } catch (err) { + if (ops.debug) console.log(err); + hasError = true; + } + expect(ops.expect.error ?? false).toBe(hasError); + expect(ops.expect.streams ?? 0).toBe(streamCount); + } + + for (const t of ops.types) { + const builder = makeBaseEmbedProviders().addSource(ops.source).addEmbed(ops.embed); + if (t === 'standard') {} + else if (t === 'ip:standard') + builder.enableConsistentIpForRequests(); + else if (t === 'proxied') { + if (!process.env.MOVIE_WEB_PROXY_URL) + throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); + builder.setProxiedFetcher(makeSimpleProxyFetcher(process.env.MOVIE_WEB_PROXY_URL, fetch)); + } + const providers = builder.build(); + try { + const embeds = await gatherEmbeds(providers); + embeds.forEach((embed, i) => { + it(`${t} - embed ${i}`, async () => { + await runTest(providers, embed.url); + }) + }) + } catch (err) { + it(`${t} - embed ??`, () => { + throw new Error("Failed to get streams: " + err); + }) + } + } + }) + }) + }) +} diff --git a/src/__test__/providers/embeds.test.ts b/src/__test__/providers/embeds.test.ts index 5775a91..d9d1d23 100644 --- a/src/__test__/providers/embeds.test.ts +++ b/src/__test__/providers/embeds.test.ts @@ -1,17 +1,118 @@ -import { febboxMp4Scraper } from "@/providers/embeds/febbox/mp4"; -import { testEmbed } from "./providerUtils"; import dotenv from 'dotenv'; +import { febboxMp4Scraper } from "@/providers/embeds/febbox/mp4"; +import { testEmbed } from "./embedUtils"; +import { showboxScraper } from "@/providers/sources/showbox"; +import { testMedia } from "./testMedia"; +import { flixhqScraper } from "@/providers/sources/flixhq"; +import { upcloudScraper } from "@/providers/embeds/upcloud"; +import { goMoviesScraper } from "@/providers/sources/gomovies"; +import { smashyStreamScraper } from "@/providers/sources/smashystream"; +import { smashyStreamDScraper } from "@/providers/embeds/smashystream/dued"; +import { vidsrcembedScraper } from '@/providers/embeds/vidsrc'; +import { vidsrcScraper } from '@/providers/sources/vidsrc'; +import { vidSrcToScraper } from '@/providers/sources/vidsrcto'; +import { vidplayScraper } from '@/providers/embeds/vidplay'; +import { fileMoonScraper } from '@/providers/embeds/filemoon'; +import { zoechipScraper } from '@/providers/sources/zoechip'; +import { mixdropScraper } from '@/providers/embeds/mixdrop'; dotenv.config(); testEmbed({ embed: febboxMp4Scraper, - testUrls: [ - '/show/16448/1/1', - '/movie/27769//' - ], + source: showboxScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], types: ['standard', 'proxied'], expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: upcloudScraper, + source: flixhqScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: upcloudScraper, + source: goMoviesScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: smashyStreamDScraper, + source: smashyStreamScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: vidsrcembedScraper, + source: vidsrcScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: vidplayScraper, + source: vidSrcToScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: fileMoonScraper, + source: vidSrcToScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: upcloudScraper, + source: zoechipScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 2, + streams: 1, + } +}) + +testEmbed({ + embed: mixdropScraper, + source: zoechipScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 2, streams: 1, } }) diff --git a/src/__test__/providers/providerUtils.ts b/src/__test__/providers/providerUtils.ts index 65fe710..f5d126c 100644 --- a/src/__test__/providers/providerUtils.ts +++ b/src/__test__/providers/providerUtils.ts @@ -1,5 +1,5 @@ import { ScrapeMedia } from "@/entrypoint/utils/media"; -import { Embed, Sourcerer } from "@/providers/base"; +import { Embed, Sourcerer, SourcererEmbed } from "@/providers/base"; import { buildProviders } from "@/entrypoint/builder"; import { describe, expect, it } from "vitest"; import { makeStandardFetcher } from "@/fetchers/standardFetch"; @@ -24,41 +24,26 @@ export interface TestSourceOptions { } } -export interface TestEmbedOptions { - embed: Embed; - testUrls: string[]; - types: TestTypes[]; - debug?: boolean; - expect: { - streams?: number; - error?: boolean; - } -} - function makeBaseProviders() { - const builder = makeBaseEmbedProviders(); - const embeds = getBuiltinEmbeds(); - embeds.forEach(embed => builder.addEmbed(embed)); - return builder; -} - -function makeBaseEmbedProviders() { const builder = buildProviders() .setTarget(targets.ANY) .setFetcher(makeStandardFetcher(fetch)); + const embeds = getBuiltinEmbeds(); + embeds.forEach(embed => builder.addEmbed(embed)); return builder; } export function testSource(ops: TestSourceOptions) { if (ops.testSuite.length === 0) throw new Error("Test suite must have at least one test"); describe(`source:${ops.source.id}`, () => { - ops.testSuite.forEach((test, i) => { - describe(`test ${i}`, () => { + ops.testSuite.forEach((test) => { + describe(`test ${test.title}`, () => { async function runTest(providers: ProviderControls) { let hasNotFound = false; let hasError = false; let streamCount = 0; let embedCount = 0; + let embeds = []; try { const result = await providers.runSourceScraper({ id: ops.source.id, @@ -115,61 +100,3 @@ export function testSource(ops: TestSourceOptions) { }) }) } - -export function testEmbed(ops: TestEmbedOptions) { - if (ops.testUrls.length === 0) throw new Error("Test urls must have at least one url"); - describe(`embed:${ops.embed.id}`, () => { - ops.testUrls.forEach((test, i) => { - describe(`test ${i}`, () => { - async function runTest(providers: ProviderControls) { - let hasError = false; - let streamCount = 0; - try { - const result = await providers.runEmbedScraper({ - id: ops.embed.id, - url: test, - }) - if (ops.debug) console.log(result); - streamCount = (result.stream ?? []).length; - } catch (err) { - if (ops.debug) console.log(err); - hasError = true; - } - expect(ops.expect.error ?? false).toBe(hasError); - expect(ops.expect.streams ?? 0).toBe(streamCount); - } - - if (ops.types.includes('standard')) { - it(`standard`, async () => { - const providers = makeBaseEmbedProviders() - .addEmbed(ops.embed) - .build(); - await runTest(providers); - }) - } - - if (ops.types.includes('ip:standard')) { - it(`standard:ip`, async () => { - const providers = makeBaseEmbedProviders() - .addEmbed(ops.embed) - .enableConsistentIpForRequests() - .build(); - await runTest(providers); - }) - } - - if (ops.types.includes('proxied')) { - it(`proxied`, async () => { - if (!process.env.MOVIE_WEB_PROXY_URL) - throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); - const providers = makeBaseEmbedProviders() - .addEmbed(ops.embed) - .setProxiedFetcher(makeSimpleProxyFetcher(process.env.MOVIE_WEB_PROXY_URL, fetch)) - .build(); - await runTest(providers); - }) - } - }); - }) - }) -} diff --git a/src/__test__/providers/providers.test.ts b/src/__test__/providers/providers.test.ts index 0f29dc1..ecac090 100644 --- a/src/__test__/providers/providers.test.ts +++ b/src/__test__/providers/providers.test.ts @@ -3,6 +3,13 @@ import { lookmovieScraper } from "@/providers/sources/lookmovie"; import { testMedia } from "./testMedia"; import { showboxScraper } from "@/providers/sources/showbox"; import dotenv from 'dotenv'; +import { flixhqScraper } from "@/providers/sources/flixhq"; +import { goMoviesScraper } from "@/providers/sources/gomovies"; +import { smashyStreamScraper } from "@/providers/sources/smashystream"; +import { vidsrcScraper } from "@/providers/sources/vidsrc"; +import { vidSrcToScraper } from "@/providers/sources/vidsrcto"; +import { zoechipScraper } from "@/providers/sources/zoechip"; +import { remotestreamScraper } from "@/providers/sources/remotestream"; dotenv.config(); @@ -23,3 +30,66 @@ testSource({ embeds: 1, } }) + +testSource({ + source: flixhqScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + } +}) + +testSource({ + source: goMoviesScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + } +}) + +testSource({ + source: smashyStreamScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + } +}) + +testSource({ + source: vidsrcScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + } +}) + +testSource({ + source: vidSrcToScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 2, + } +}) + +testSource({ + source: zoechipScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 3, + } +}) + +testSource({ + source: remotestreamScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + streams: 1, + } +}) From f3e4786b72ceca26a38752cc288c4a78ce44c485 Mon Sep 17 00:00:00 2001 From: Jorrin Date: Fri, 19 Jan 2024 20:04:50 +0100 Subject: [PATCH 11/22] Remove equal signs inside base64 encoded string --- src/providers/embeds/vidsrc.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/providers/embeds/vidsrc.ts b/src/providers/embeds/vidsrc.ts index cd47e21..9da5a9f 100644 --- a/src/providers/embeds/vidsrc.ts +++ b/src/providers/embeds/vidsrc.ts @@ -18,7 +18,8 @@ export const vidsrcembedScraper = makeEmbed({ const match = html .match(hlsURLRegex)?.[1] ?.replace(/(\/\/\S+?=)/g, '') - .replace('#2', ''); + .replace('#2', '') + .replace(/=/g, ''); if (!match) throw new Error('Unable to find HLS playlist'); const finalUrl = atob(match); From b13578cefadd5639c7e2b0a6a7737ae3f615458d Mon Sep 17 00:00:00 2001 From: Jorrin Date: Fri, 19 Jan 2024 20:13:44 +0100 Subject: [PATCH 12/22] improve regex --- src/providers/embeds/vidsrc.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/providers/embeds/vidsrc.ts b/src/providers/embeds/vidsrc.ts index 9da5a9f..93d55c5 100644 --- a/src/providers/embeds/vidsrc.ts +++ b/src/providers/embeds/vidsrc.ts @@ -15,11 +15,7 @@ export const vidsrcembedScraper = makeEmbed({ }, }); - const match = html - .match(hlsURLRegex)?.[1] - ?.replace(/(\/\/\S+?=)/g, '') - .replace('#2', '') - .replace(/=/g, ''); + const match = html.match(hlsURLRegex)?.[1]?.replace(/(\/\/\S+?=)|#2|=/g, ''); if (!match) throw new Error('Unable to find HLS playlist'); const finalUrl = atob(match); From ec4f2d02edf4bb7c1c5a95db1521598e12c2a58a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Jan 2024 05:46:23 +0000 Subject: [PATCH 13/22] Bump vite from 4.5.1 to 4.5.2 in /.docs Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 4.5.1 to 4.5.2. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/v4.5.2/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v4.5.2/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-type: indirect ... Signed-off-by: dependabot[bot] --- .docs/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.docs/package-lock.json b/.docs/package-lock.json index 3a9fa95..11e9e1a 100644 --- a/.docs/package-lock.json +++ b/.docs/package-lock.json @@ -17286,9 +17286,9 @@ "dev": true }, "node_modules/vite": { - "version": "4.5.1", - "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.1.tgz", - "integrity": "sha512-AXXFaAJ8yebyqzoNB9fu2pHoo/nWX+xZlaRwoeYUxEqBO+Zj4msE5G+BhGBll9lYEKv9Hfks52PAF2X7qDYXQA==", + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.2.tgz", + "integrity": "sha512-tBCZBNSBbHQkaGyhGCDUGqeo2ph8Fstyp6FMSvTtsXeZSPpSMGlviAOav2hxVTqFcx8Hj/twtWKsMJXNY0xI8w==", "dev": true, "dependencies": { "esbuild": "^0.18.10", From efa4857b20a8f86ab9a88d6c4a5055bbf029cbc5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Jan 2024 05:53:00 +0000 Subject: [PATCH 14/22] Bump vite from 4.5.1 to 4.5.2 Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 4.5.1 to 4.5.2. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/v4.5.2/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v4.5.2/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- package-lock.json | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index f38bd59..9954f93 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2848,6 +2848,20 @@ "dev": true, "license": "ISC" }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/function-bind": { "version": "1.1.2", "dev": true, @@ -5517,9 +5531,10 @@ } }, "node_modules/vite": { - "version": "4.5.1", + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.2.tgz", + "integrity": "sha512-tBCZBNSBbHQkaGyhGCDUGqeo2ph8Fstyp6FMSvTtsXeZSPpSMGlviAOav2hxVTqFcx8Hj/twtWKsMJXNY0xI8w==", "dev": true, - "license": "MIT", "dependencies": { "esbuild": "^0.18.10", "postcss": "^8.4.27", From f2073e89d95f5669b2312e4cf356c97107917bf5 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sat, 20 Jan 2024 14:18:44 +0100 Subject: [PATCH 15/22] Fix remotestream and linting error --- src/providers/embeds/vidplay/index.ts | 1 - src/providers/sources/remotestream.ts | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/providers/embeds/vidplay/index.ts b/src/providers/embeds/vidplay/index.ts index 9d205e1..48af6c1 100644 --- a/src/providers/embeds/vidplay/index.ts +++ b/src/providers/embeds/vidplay/index.ts @@ -1,4 +1,3 @@ -import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; diff --git a/src/providers/sources/remotestream.ts b/src/providers/sources/remotestream.ts index f0a0ae8..fa2efc4 100644 --- a/src/providers/sources/remotestream.ts +++ b/src/providers/sources/remotestream.ts @@ -19,8 +19,8 @@ export const remotestreamScraper = makeSourcerer({ const playlistLink = `${remotestreamBase}/Shows/${ctx.media.tmdbId}/${seasonNumber}/${episodeNumber}/${episodeNumber}.m3u8`; ctx.progress(30); - const streamRes = await ctx.fetcher.full(playlistLink, { - method: 'HEAD', + const streamRes = await ctx.proxiedFetcher.full(playlistLink, { + method: 'GET', readHeaders: ['content-type'], headers: { Referer: referer, @@ -51,8 +51,8 @@ export const remotestreamScraper = makeSourcerer({ const playlistLink = `${remotestreamBase}/Movies/${ctx.media.tmdbId}/${ctx.media.tmdbId}.m3u8`; ctx.progress(30); - const streamRes = await ctx.fetcher.full(playlistLink, { - method: 'HEAD', + const streamRes = await ctx.proxiedFetcher.full(playlistLink, { + method: 'GET', readHeaders: ['content-type'], headers: { Referer: referer, From 2df4f10f642287a00a01bc7bd0246f066de7814d Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sat, 20 Jan 2024 17:54:44 +0100 Subject: [PATCH 16/22] Version bump + lookmovie reorder --- package.json | 2 +- src/providers/sources/lookmovie/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 4047136..0cd0ced 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@movie-web/providers", - "version": "2.0.5", + "version": "2.1.0", "description": "Package that contains all the providers of movie-web", "main": "./lib/index.umd.js", "types": "./lib/index.d.ts", diff --git a/src/providers/sources/lookmovie/index.ts b/src/providers/sources/lookmovie/index.ts index 8611373..6fbe6de 100644 --- a/src/providers/sources/lookmovie/index.ts +++ b/src/providers/sources/lookmovie/index.ts @@ -32,7 +32,7 @@ async function universalScraper(ctx: MovieScrapeContext | ShowScrapeContext): Pr export const lookmovieScraper = makeSourcerer({ id: 'lookmovie', name: 'LookMovie', - rank: 1, + rank: 700, flags: [flags.IP_LOCKED], scrapeShow: universalScraper, scrapeMovie: universalScraper, From 09cfafcaa265a977936f8640b56163f9046fd992 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sat, 20 Jan 2024 18:02:49 +0100 Subject: [PATCH 17/22] update changelog --- .docs/content/1.get-started/4.changelog.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.docs/content/1.get-started/4.changelog.md b/.docs/content/1.get-started/4.changelog.md index 239ed6b..a19930a 100644 --- a/.docs/content/1.get-started/4.changelog.md +++ b/.docs/content/1.get-started/4.changelog.md @@ -2,6 +2,12 @@ title: 'Changelog' --- +# Version 2.1.0 + - Add preferedHeaders to most sources + - Add CF_BLOCKED flag to sources that have blocked cloudflare API's + - Fix vidsrc sometimes having an equal sign where it shouldnt + - Increase ranking of lookmovie + # Version 2.0.5 - Disable subtitles for febbox-mp4. As their endpoint doesn't work anymore. From e5d9a3b23cdf0215fe26e9635ba3b9a8cc5c1728 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Sat, 20 Jan 2024 18:03:39 +0100 Subject: [PATCH 18/22] add a new line to changelog --- .docs/content/1.get-started/4.changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.docs/content/1.get-started/4.changelog.md b/.docs/content/1.get-started/4.changelog.md index a19930a..518a414 100644 --- a/.docs/content/1.get-started/4.changelog.md +++ b/.docs/content/1.get-started/4.changelog.md @@ -7,6 +7,7 @@ title: 'Changelog' - Add CF_BLOCKED flag to sources that have blocked cloudflare API's - Fix vidsrc sometimes having an equal sign where it shouldnt - Increase ranking of lookmovie + - Re-enabled subtitles for febbox-mp4 # Version 2.0.5 - Disable subtitles for febbox-mp4. As their endpoint doesn't work anymore. From 5adca068a8248a5db41f768ee636597f48a932bd Mon Sep 17 00:00:00 2001 From: Jorrin <43169049+JorrinKievit@users.noreply.github.com> Date: Wed, 24 Jan 2024 17:32:34 +0100 Subject: [PATCH 19/22] replace vidplay decryption keys and domain - Massive thanks to @Ciarands! --- package-lock.json | 4 ++-- src/providers/embeds/vidplay/common.ts | 8 +++----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/package-lock.json b/package-lock.json index d8a8961..3facef2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@movie-web/providers", - "version": "2.0.5", + "version": "2.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@movie-web/providers", - "version": "2.0.5", + "version": "2.1.0", "license": "MIT", "dependencies": { "cheerio": "^1.0.0-rc.12", diff --git a/src/providers/embeds/vidplay/common.ts b/src/providers/embeds/vidplay/common.ts index e3172f7..3eb19a9 100644 --- a/src/providers/embeds/vidplay/common.ts +++ b/src/providers/embeds/vidplay/common.ts @@ -2,16 +2,14 @@ import { makeFullUrl } from '@/fetchers/common'; import { decodeData } from '@/providers/sources/vidsrcto/common'; import { EmbedScrapeContext } from '@/utils/context'; -export const vidplayBase = 'https://vidplay.site'; -export const referer = 'https://vidplay.online/'; +export const vidplayBase = 'https://vidplay.online'; +export const referer = `${vidplayBase}/`; // This file is based on https://github.com/Ciarands/vidsrc-to-resolver/blob/dffa45e726a4b944cb9af0c9e7630476c93c0213/vidsrc.py#L16 // Full credits to @Ciarands! export const getDecryptionKeys = async (ctx: EmbedScrapeContext): Promise => { - const res = await ctx.fetcher( - 'https://raw.githubusercontent.com/Claudemirovsky/worstsource-keys/keys/keys.json', - ); + const res = await ctx.fetcher('https://raw.githubusercontent.com/Ciarands/vidsrc-keys/main/keys.json'); return JSON.parse(res); }; From 89c7f49be2c140f8a659e4e078850b451e473088 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 24 Jan 2024 20:06:08 +0100 Subject: [PATCH 20/22] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0cd0ced..25c4d07 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@movie-web/providers", - "version": "2.1.0", + "version": "2.1.1", "description": "Package that contains all the providers of movie-web", "main": "./lib/index.umd.js", "types": "./lib/index.d.ts", From 3c2e3bc07631189d0a14cd50dc5d1a474304f7e3 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Wed, 24 Jan 2024 20:08:43 +0100 Subject: [PATCH 21/22] Update 4.changelog.md --- .docs/content/1.get-started/4.changelog.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.docs/content/1.get-started/4.changelog.md b/.docs/content/1.get-started/4.changelog.md index 518a414..f03e407 100644 --- a/.docs/content/1.get-started/4.changelog.md +++ b/.docs/content/1.get-started/4.changelog.md @@ -2,6 +2,9 @@ title: 'Changelog' --- +# Version 2.1.1 + - Fixed vidplay decryption keys being wrong and switched the domain to one that works + # Version 2.1.0 - Add preferedHeaders to most sources - Add CF_BLOCKED flag to sources that have blocked cloudflare API's From 4591bcbc2e26470f65f9c3d25cd0f1d882fe139c Mon Sep 17 00:00:00 2001 From: Ciarands Date: Fri, 26 Jan 2024 23:16:37 +0000 Subject: [PATCH 22/22] Fix vidsrc.me base64 decoding --- src/providers/embeds/vidsrc.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/providers/embeds/vidsrc.ts b/src/providers/embeds/vidsrc.ts index 93d55c5..069c2f6 100644 --- a/src/providers/embeds/vidsrc.ts +++ b/src/providers/embeds/vidsrc.ts @@ -4,6 +4,14 @@ import { makeEmbed } from '@/providers/base'; const hlsURLRegex = /file:"(.*?)"/; const setPassRegex = /var pass_path = "(.*set_pass\.php.*)";/; +function formatHlsB64(data: string): string { + const encodedB64 = data.replace(/\/@#@\/[^=/]+==/g, ''); + if (encodedB64.match(/\/@#@\/[^=/]+==/)) { + return formatHlsB64(encodedB64); + } + return encodedB64; +} + export const vidsrcembedScraper = makeEmbed({ id: 'vidsrcembed', // VidSrc is both a source and an embed host name: 'VidSrc', @@ -15,10 +23,12 @@ export const vidsrcembedScraper = makeEmbed({ }, }); - const match = html.match(hlsURLRegex)?.[1]?.replace(/(\/\/\S+?=)|#2|=/g, ''); - if (!match) throw new Error('Unable to find HLS playlist'); - const finalUrl = atob(match); - + // When this eventually breaks see the player js @ pjs_main.js + // If you know what youre doing and are slightly confused about how to reverse this feel free to reach out to ciaran_ds on discord with any queries + let hlsMatch = html.match(hlsURLRegex)?.[1]?.slice(2); + if (!hlsMatch) throw new Error('Unable to find HLS playlist'); + hlsMatch = formatHlsB64(hlsMatch); + const finalUrl = atob(hlsMatch); if (!finalUrl.includes('.m3u8')) throw new Error('Unable to find HLS playlist'); let setPassLink = html.match(setPassRegex)?.[1];