diff --git a/.docs/content/1.get-started/4.changelog.md b/.docs/content/1.get-started/4.changelog.md index 239ed6b..f03e407 100644 --- a/.docs/content/1.get-started/4.changelog.md +++ b/.docs/content/1.get-started/4.changelog.md @@ -2,6 +2,16 @@ title: 'Changelog' --- +# Version 2.1.1 + - Fixed vidplay decryption keys being wrong and switched the domain to one that works + +# Version 2.1.0 + - Add preferedHeaders to most sources + - Add CF_BLOCKED flag to sources that have blocked cloudflare API's + - Fix vidsrc sometimes having an equal sign where it shouldnt + - Increase ranking of lookmovie + - Re-enabled subtitles for febbox-mp4 + # Version 2.0.5 - Disable subtitles for febbox-mp4. As their endpoint doesn't work anymore. diff --git a/.docs/package-lock.json b/.docs/package-lock.json index 3a9fa95..11e9e1a 100644 --- a/.docs/package-lock.json +++ b/.docs/package-lock.json @@ -17286,9 +17286,9 @@ "dev": true }, "node_modules/vite": { - "version": "4.5.1", - "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.1.tgz", - "integrity": "sha512-AXXFaAJ8yebyqzoNB9fu2pHoo/nWX+xZlaRwoeYUxEqBO+Zj4msE5G+BhGBll9lYEKv9Hfks52PAF2X7qDYXQA==", + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.2.tgz", + "integrity": "sha512-tBCZBNSBbHQkaGyhGCDUGqeo2ph8Fstyp6FMSvTtsXeZSPpSMGlviAOav2hxVTqFcx8Hj/twtWKsMJXNY0xI8w==", "dev": true, "dependencies": { "esbuild": "^0.18.10", diff --git a/package-lock.json b/package-lock.json index f38bd59..3facef2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@movie-web/providers", - "version": "2.0.5", + "version": "2.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@movie-web/providers", - "version": "2.0.5", + "version": "2.1.0", "license": "MIT", "dependencies": { "cheerio": "^1.0.0-rc.12", @@ -26,6 +26,7 @@ "@typescript-eslint/parser": "^5.60.0", "@vitest/coverage-v8": "^0.34.3", "commander": "^11.0.0", + "cross-env": "^7.0.3", "dotenv": "^16.3.1", "enquirer": "^2.4.1", "eslint": "^8.30.0", @@ -1794,6 +1795,24 @@ "dev": true, "license": "MIT" }, + "node_modules/cross-env": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-env/-/cross-env-7.0.3.tgz", + "integrity": "sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.1" + }, + "bin": { + "cross-env": "src/bin/cross-env.js", + "cross-env-shell": "src/bin/cross-env-shell.js" + }, + "engines": { + "node": ">=10.14", + "npm": ">=6", + "yarn": ">=1" + } + }, "node_modules/cross-fetch": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.0.0.tgz", @@ -2848,6 +2867,20 @@ "dev": true, "license": "ISC" }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/function-bind": { "version": "1.1.2", "dev": true, @@ -5517,9 +5550,10 @@ } }, "node_modules/vite": { - "version": "4.5.1", + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/vite/-/vite-4.5.2.tgz", + "integrity": "sha512-tBCZBNSBbHQkaGyhGCDUGqeo2ph8Fstyp6FMSvTtsXeZSPpSMGlviAOav2hxVTqFcx8Hj/twtWKsMJXNY0xI8w==", "dev": true, - "license": "MIT", "dependencies": { "esbuild": "^0.18.10", "postcss": "^8.4.27", diff --git a/package.json b/package.json index d4e7626..25c4d07 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@movie-web/providers", - "version": "2.0.5", + "version": "2.1.1", "description": "Package that contains all the providers of movie-web", "main": "./lib/index.umd.js", "types": "./lib/index.d.ts", @@ -38,6 +38,7 @@ "cli": "ts-node ./src/dev-cli/index.ts", "test": "vitest run", "test:watch": "vitest", + "test:providers": "cross-env MW_TEST_PROVIDERS=true vitest run --reporter verbose", "test:integration": "node ./tests/cjs && node ./tests/esm && node ./tests/browser", "test:coverage": "vitest run --coverage", "lint": "eslint --ext .ts,.js src/", @@ -55,6 +56,7 @@ "@typescript-eslint/parser": "^5.60.0", "@vitest/coverage-v8": "^0.34.3", "commander": "^11.0.0", + "cross-env": "^7.0.3", "dotenv": "^16.3.1", "enquirer": "^2.4.1", "eslint": "^8.30.0", diff --git a/src/__test__/providers/embedUtils.ts b/src/__test__/providers/embedUtils.ts new file mode 100644 index 0000000..3ba88a3 --- /dev/null +++ b/src/__test__/providers/embedUtils.ts @@ -0,0 +1,90 @@ +import { buildProviders } from "@/entrypoint/builder"; +import { ScrapeMedia } from "@/entrypoint/utils/media"; +import { targets } from "@/entrypoint/utils/targets"; +import { makeStandardFetcher } from "@/fetchers/standardFetch"; +import { Embed, Sourcerer, SourcererEmbed } from "@/providers/base"; +import { TestTypes } from "./providerUtils"; +import { describe, expect, it } from "vitest"; +import { ProviderControls } from "@/entrypoint/controls"; +import { makeSimpleProxyFetcher } from "@/fetchers/simpleProxy"; + +export interface TestEmbedOptions { + embed: Embed; + source: Sourcerer; + testSuite: ScrapeMedia[]; + types: TestTypes[]; + debug?: boolean; + expect: { + embeds: number; + streams?: number; + error?: boolean; + } +} + +function makeBaseEmbedProviders() { + const builder = buildProviders() + .setTarget(targets.ANY) + .setFetcher(makeStandardFetcher(fetch)); + return builder; +} + +export function testEmbed(ops: TestEmbedOptions) { + if (ops.testSuite.length === 0) throw new Error("Test suite must have at least one test"); + describe(`embed:${ops.source.id}:${ops.embed.id}`, () => { + ops.testSuite.forEach((test) => { + describe(`test ${test.title}`, async () => { + async function gatherEmbeds(providers: ProviderControls): Promise { + const results = await providers.runSourceScraper({ + id: ops.source.id, + media: test, + }) + if (results.embeds.length !== ops.expect.embeds) throw new Error(`Embeds don't match expected amount of embeds (${ops.source.id}, ${ops.embed.id}, got ${results.embeds.length} but expected ${ops.expect.embeds})`); + return results.embeds; + } + + async function runTest(providers: ProviderControls, embedUrl: string) { + let hasError = false; + let streamCount = 0; + try { + const result = await providers.runEmbedScraper({ + id: ops.embed.id, + url: embedUrl, + }) + if (ops.debug) console.log(result); + streamCount = (result.stream ?? []).length; + } catch (err) { + if (ops.debug) console.log(err); + hasError = true; + } + expect(ops.expect.error ?? false).toBe(hasError); + expect(ops.expect.streams ?? 0).toBe(streamCount); + } + + for (const t of ops.types) { + const builder = makeBaseEmbedProviders().addSource(ops.source).addEmbed(ops.embed); + if (t === 'standard') {} + else if (t === 'ip:standard') + builder.enableConsistentIpForRequests(); + else if (t === 'proxied') { + if (!process.env.MOVIE_WEB_PROXY_URL) + throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); + builder.setProxiedFetcher(makeSimpleProxyFetcher(process.env.MOVIE_WEB_PROXY_URL, fetch)); + } + const providers = builder.build(); + try { + const embeds = await gatherEmbeds(providers); + embeds.forEach((embed, i) => { + it(`${t} - embed ${i}`, async () => { + await runTest(providers, embed.url); + }) + }) + } catch (err) { + it(`${t} - embed ??`, () => { + throw new Error("Failed to get streams: " + err); + }) + } + } + }) + }) + }) +} diff --git a/src/__test__/providers/embeds.test.ts b/src/__test__/providers/embeds.test.ts new file mode 100644 index 0000000..d9d1d23 --- /dev/null +++ b/src/__test__/providers/embeds.test.ts @@ -0,0 +1,118 @@ +import dotenv from 'dotenv'; +import { febboxMp4Scraper } from "@/providers/embeds/febbox/mp4"; +import { testEmbed } from "./embedUtils"; +import { showboxScraper } from "@/providers/sources/showbox"; +import { testMedia } from "./testMedia"; +import { flixhqScraper } from "@/providers/sources/flixhq"; +import { upcloudScraper } from "@/providers/embeds/upcloud"; +import { goMoviesScraper } from "@/providers/sources/gomovies"; +import { smashyStreamScraper } from "@/providers/sources/smashystream"; +import { smashyStreamDScraper } from "@/providers/embeds/smashystream/dued"; +import { vidsrcembedScraper } from '@/providers/embeds/vidsrc'; +import { vidsrcScraper } from '@/providers/sources/vidsrc'; +import { vidSrcToScraper } from '@/providers/sources/vidsrcto'; +import { vidplayScraper } from '@/providers/embeds/vidplay'; +import { fileMoonScraper } from '@/providers/embeds/filemoon'; +import { zoechipScraper } from '@/providers/sources/zoechip'; +import { mixdropScraper } from '@/providers/embeds/mixdrop'; + +dotenv.config(); + +testEmbed({ + embed: febboxMp4Scraper, + source: showboxScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: upcloudScraper, + source: flixhqScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: upcloudScraper, + source: goMoviesScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: smashyStreamDScraper, + source: smashyStreamScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: vidsrcembedScraper, + source: vidsrcScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: vidplayScraper, + source: vidSrcToScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: fileMoonScraper, + source: vidSrcToScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + streams: 1, + } +}) + +testEmbed({ + embed: upcloudScraper, + source: zoechipScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 2, + streams: 1, + } +}) + +testEmbed({ + embed: mixdropScraper, + source: zoechipScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 2, + streams: 1, + } +}) diff --git a/src/__test__/providers/providerUtils.ts b/src/__test__/providers/providerUtils.ts new file mode 100644 index 0000000..f5d126c --- /dev/null +++ b/src/__test__/providers/providerUtils.ts @@ -0,0 +1,102 @@ +import { ScrapeMedia } from "@/entrypoint/utils/media"; +import { Embed, Sourcerer, SourcererEmbed } from "@/providers/base"; +import { buildProviders } from "@/entrypoint/builder"; +import { describe, expect, it } from "vitest"; +import { makeStandardFetcher } from "@/fetchers/standardFetch"; +import { ProviderControls } from "@/entrypoint/controls"; +import { NotFoundError } from "@/utils/errors"; +import { targets } from "@/entrypoint/utils/targets"; +import { getBuiltinEmbeds } from "@/entrypoint/providers"; +import { makeSimpleProxyFetcher } from "@/fetchers/simpleProxy"; + +export type TestTypes = 'standard' | 'ip:standard' | 'proxied'; + +export interface TestSourceOptions { + source: Sourcerer; + testSuite: ScrapeMedia[]; + types: TestTypes[]; + debug?: boolean; + expect: { + embeds?: number; + streams?: number; + error?: boolean; + notfound?: boolean; + } +} + +function makeBaseProviders() { + const builder = buildProviders() + .setTarget(targets.ANY) + .setFetcher(makeStandardFetcher(fetch)); + const embeds = getBuiltinEmbeds(); + embeds.forEach(embed => builder.addEmbed(embed)); + return builder; +} + +export function testSource(ops: TestSourceOptions) { + if (ops.testSuite.length === 0) throw new Error("Test suite must have at least one test"); + describe(`source:${ops.source.id}`, () => { + ops.testSuite.forEach((test) => { + describe(`test ${test.title}`, () => { + async function runTest(providers: ProviderControls) { + let hasNotFound = false; + let hasError = false; + let streamCount = 0; + let embedCount = 0; + let embeds = []; + try { + const result = await providers.runSourceScraper({ + id: ops.source.id, + media: test, + }) + if (ops.debug) console.log(result); + streamCount = (result.stream ?? []).length; + embedCount = result.embeds.length; + } catch (err) { + if (ops.debug) console.log(err); + if (err instanceof NotFoundError) + hasNotFound = true; + else + hasError = true; + } + expect(ops.expect.error ?? false).toBe(hasError); + expect(ops.expect.notfound ?? false).toBe(hasNotFound); + expect(ops.expect.streams ?? 0).toBe(streamCount); + expect(ops.expect.embeds ?? 0).toBe(embedCount); + } + + if (ops.types.includes('standard')) { + it(`standard`, async () => { + const providers = makeBaseProviders() + .addSource(ops.source) + .build(); + await runTest(providers); + }) + } + + if (ops.types.includes('ip:standard')) { + it(`standard:ip`, async () => { + const providers = makeBaseProviders() + .addSource(ops.source) + .enableConsistentIpForRequests() + .build(); + await runTest(providers); + }) + } + + if (ops.types.includes('proxied')) { + it(`proxied`, async () => { + if (!process.env.MOVIE_WEB_PROXY_URL) + throw new Error("Cant use proxied test without setting MOVIE_WEB_PROXY_URL env"); + const providers = makeBaseProviders() + .addSource(ops.source) + .setProxiedFetcher(makeSimpleProxyFetcher(process.env.MOVIE_WEB_PROXY_URL, fetch)) + .build(); + await runTest(providers); + }) + } + + }) + }) + }) +} diff --git a/src/__test__/providers/providers.test.ts b/src/__test__/providers/providers.test.ts new file mode 100644 index 0000000..ecac090 --- /dev/null +++ b/src/__test__/providers/providers.test.ts @@ -0,0 +1,95 @@ +import { testSource } from "./providerUtils"; +import { lookmovieScraper } from "@/providers/sources/lookmovie"; +import { testMedia } from "./testMedia"; +import { showboxScraper } from "@/providers/sources/showbox"; +import dotenv from 'dotenv'; +import { flixhqScraper } from "@/providers/sources/flixhq"; +import { goMoviesScraper } from "@/providers/sources/gomovies"; +import { smashyStreamScraper } from "@/providers/sources/smashystream"; +import { vidsrcScraper } from "@/providers/sources/vidsrc"; +import { vidSrcToScraper } from "@/providers/sources/vidsrcto"; +import { zoechipScraper } from "@/providers/sources/zoechip"; +import { remotestreamScraper } from "@/providers/sources/remotestream"; + +dotenv.config(); + +testSource({ + source: lookmovieScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['ip:standard'], + expect: { + streams: 1, + } +}) + +testSource({ + source: showboxScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + } +}) + +testSource({ + source: flixhqScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + } +}) + +testSource({ + source: goMoviesScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + } +}) + +testSource({ + source: smashyStreamScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + } +}) + +testSource({ + source: vidsrcScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 1, + } +}) + +testSource({ + source: vidSrcToScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 2, + } +}) + +testSource({ + source: zoechipScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + embeds: 3, + } +}) + +testSource({ + source: remotestreamScraper, + testSuite: [testMedia.arcane, testMedia.hamilton], + types: ['standard', 'proxied'], + expect: { + streams: 1, + } +}) diff --git a/src/__test__/providers/testMedia.ts b/src/__test__/providers/testMedia.ts new file mode 100644 index 0000000..0e0d050 --- /dev/null +++ b/src/__test__/providers/testMedia.ts @@ -0,0 +1,30 @@ +import { ScrapeMedia } from "@/entrypoint/utils/media"; + +function makeMedia(media: ScrapeMedia): ScrapeMedia { + return media; +} + +export const testMedia = { + arcane: makeMedia({ + type: "show", + title: "Arcane", + tmdbId: "94605", + releaseYear: 2021, + episode: { + number: 1, + tmdbId: '1953812', + }, + season: { + number: 1, + tmdbId: '134187', + }, + imdbId: 'tt11126994' + }), + hamilton: makeMedia({ + type: 'movie', + tmdbId: '556574', + imdbId: 'tt8503618', + releaseYear: 2020, + title: 'Hamilton' + }) +} diff --git a/src/__test__/fetchers/body.test.ts b/src/__test__/standard/fetchers/body.test.ts similarity index 100% rename from src/__test__/fetchers/body.test.ts rename to src/__test__/standard/fetchers/body.test.ts diff --git a/src/__test__/fetchers/common.test.ts b/src/__test__/standard/fetchers/common.test.ts similarity index 100% rename from src/__test__/fetchers/common.test.ts rename to src/__test__/standard/fetchers/common.test.ts diff --git a/src/__test__/fetchers/simpleProxy.test.ts b/src/__test__/standard/fetchers/simpleProxy.test.ts similarity index 100% rename from src/__test__/fetchers/simpleProxy.test.ts rename to src/__test__/standard/fetchers/simpleProxy.test.ts diff --git a/src/__test__/fetchers/standard.test.ts b/src/__test__/standard/fetchers/standard.test.ts similarity index 100% rename from src/__test__/fetchers/standard.test.ts rename to src/__test__/standard/fetchers/standard.test.ts diff --git a/src/__test__/providerTests.ts b/src/__test__/standard/providerTests.ts similarity index 69% rename from src/__test__/providerTests.ts rename to src/__test__/standard/providerTests.ts index 551a3ec..f5c87b7 100644 --- a/src/__test__/providerTests.ts +++ b/src/__test__/standard/providerTests.ts @@ -2,7 +2,7 @@ import { vi } from 'vitest'; import { gatherAllEmbeds, gatherAllSources } from '@/providers/all'; -import { Embed, Sourcerer } from '@/providers/base'; +import { makeEmbed, makeSourcerer } from '@/providers/base'; export function makeProviderMocks() { const embedsMock = vi.fn, ReturnType>(); @@ -13,104 +13,104 @@ export function makeProviderMocks() { }; } -const sourceA = { +const sourceA = makeSourcerer({ id: 'a', name: 'A', rank: 1, disabled: false, flags: [], -} as Sourcerer; -const sourceB = { +}); +const sourceB = makeSourcerer({ id: 'b', name: 'B', rank: 2, disabled: false, flags: [], -} as Sourcerer; -const sourceCDisabled = { +}); +const sourceCDisabled = makeSourcerer({ id: 'c', name: 'C', rank: 3, disabled: true, flags: [], -} as Sourcerer; -const sourceAHigherRank = { +}); +const sourceAHigherRank = makeSourcerer({ id: 'a', name: 'A', rank: 100, disabled: false, flags: [], -} as Sourcerer; -const sourceGSameRankAsA = { +}); +const sourceGSameRankAsA = makeSourcerer({ id: 'g', name: 'G', rank: 1, disabled: false, flags: [], -} as Sourcerer; -const fullSourceYMovie = { +}); +const fullSourceYMovie = makeSourcerer({ id: 'y', name: 'Y', rank: 105, scrapeMovie: vi.fn(), flags: [], -} as Sourcerer; -const fullSourceYShow = { +}); +const fullSourceYShow = makeSourcerer({ id: 'y', name: 'Y', rank: 105, scrapeShow: vi.fn(), flags: [], -} as Sourcerer; -const fullSourceZBoth = { +}); +const fullSourceZBoth = makeSourcerer({ id: 'z', name: 'Z', rank: 106, scrapeMovie: vi.fn(), scrapeShow: vi.fn(), flags: [], -} as Sourcerer; +}); -const embedD = { +const embedD = makeEmbed({ id: 'd', rank: 4, disabled: false, -} as Embed; -const embedA = { +} as any); +const embedA = makeEmbed({ id: 'a', rank: 5, disabled: false, -} as Embed; -const embedEDisabled = { +} as any); +const embedEDisabled = makeEmbed({ id: 'e', rank: 6, disabled: true, -} as Embed; -const embedDHigherRank = { +} as any); +const embedDHigherRank = makeEmbed({ id: 'd', rank: 4000, disabled: false, -} as Embed; -const embedFSameRankAsA = { +} as any); +const embedFSameRankAsA = makeEmbed({ id: 'f', rank: 5, disabled: false, -} as Embed; -const embedHSameRankAsSourceA = { +} as any); +const embedHSameRankAsSourceA = makeEmbed({ id: 'h', rank: 1, disabled: false, -} as Embed; -const fullEmbedX = { +} as any); +const fullEmbedX = makeEmbed({ id: 'x', name: 'X', rank: 104, -} as Embed; -const fullEmbedZ = { +} as any); +const fullEmbedZ = makeEmbed({ id: 'z', name: 'Z', rank: 109, -} as Embed; +} as any); export const mockSources = { sourceA, diff --git a/src/__test__/providers/checks.test.ts b/src/__test__/standard/providers/checks.test.ts similarity index 98% rename from src/__test__/providers/checks.test.ts rename to src/__test__/standard/providers/checks.test.ts index 404fb31..747c240 100644 --- a/src/__test__/providers/checks.test.ts +++ b/src/__test__/standard/providers/checks.test.ts @@ -1,4 +1,4 @@ -import { mockEmbeds, mockSources } from '@/__test__/providerTests'; +import { mockEmbeds, mockSources } from '../providerTests'; import { getBuiltinEmbeds, getBuiltinSources } from '@/entrypoint/providers'; import { FeatureMap } from '@/entrypoint/utils/targets'; import { getProviders } from '@/providers/get'; diff --git a/src/__test__/runner/list.test.ts b/src/__test__/standard/runner/list.test.ts similarity index 98% rename from src/__test__/runner/list.test.ts rename to src/__test__/standard/runner/list.test.ts index 336de44..b615457 100644 --- a/src/__test__/runner/list.test.ts +++ b/src/__test__/standard/runner/list.test.ts @@ -1,4 +1,4 @@ -import { mockEmbeds, mockSources } from '@/__test__/providerTests'; +import { mockEmbeds, mockSources } from '../providerTests.ts'; import { makeProviders } from '@/entrypoint/declare'; import { targets } from '@/entrypoint/utils/targets'; import { afterEach, describe, expect, it, vi } from 'vitest'; diff --git a/src/__test__/runner/meta.test.ts b/src/__test__/standard/runner/meta.test.ts similarity index 96% rename from src/__test__/runner/meta.test.ts rename to src/__test__/standard/runner/meta.test.ts index 423a8e6..ac2fab6 100644 --- a/src/__test__/runner/meta.test.ts +++ b/src/__test__/standard/runner/meta.test.ts @@ -1,4 +1,4 @@ -import { mockEmbeds, mockSources } from '@/__test__/providerTests'; +import { mockEmbeds, mockSources } from '../providerTests.ts'; import { makeProviders } from '@/entrypoint/declare'; import { targets } from '@/entrypoint/utils/targets'; import { afterEach, describe, expect, it, vi } from 'vitest'; diff --git a/src/__test__/utils/features.test.ts b/src/__test__/standard/utils/features.test.ts similarity index 100% rename from src/__test__/utils/features.test.ts rename to src/__test__/standard/utils/features.test.ts diff --git a/src/__test__/utils/list.test.ts b/src/__test__/standard/utils/list.test.ts similarity index 100% rename from src/__test__/utils/list.test.ts rename to src/__test__/standard/utils/list.test.ts diff --git a/src/__test__/utils/valid.test.ts b/src/__test__/standard/utils/valid.test.ts similarity index 100% rename from src/__test__/utils/valid.test.ts rename to src/__test__/standard/utils/valid.test.ts diff --git a/src/entrypoint/utils/targets.ts b/src/entrypoint/utils/targets.ts index 16a02da..80988a4 100644 --- a/src/entrypoint/utils/targets.ts +++ b/src/entrypoint/utils/targets.ts @@ -5,6 +5,10 @@ export const flags = { // the stream is locked on IP, so only works if // request maker is same as player (not compatible with proxies) IP_LOCKED: 'ip-locked', + + // The source/embed is blocking cloudflare ip's + // This flag is not compatible with a proxy hosted on cloudflare + CF_BLOCKED: 'cf-blocked', } as const; export type Flags = (typeof flags)[keyof typeof flags]; diff --git a/src/providers/embeds/febbox/mp4.ts b/src/providers/embeds/febbox/mp4.ts index b69812f..1122e53 100644 --- a/src/providers/embeds/febbox/mp4.ts +++ b/src/providers/embeds/febbox/mp4.ts @@ -2,6 +2,7 @@ import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { parseInputUrl } from '@/providers/embeds/febbox/common'; import { getStreamQualities } from '@/providers/embeds/febbox/qualities'; +import { getSubtitles } from '@/providers/embeds/febbox/subtitles'; export const febboxMp4Scraper = makeEmbed({ id: 'febbox-mp4', @@ -41,7 +42,7 @@ export const febboxMp4Scraper = makeEmbed({ stream: [ { id: 'primary', - captions: [], // subtitles temporarily disabled, the endpoints are broken + captions: await getSubtitles(ctx, id, fid, type, episode, season), qualities, type: 'file', flags: [flags.CORS_ALLOWED], diff --git a/src/providers/embeds/upcloud.ts b/src/providers/embeds/upcloud.ts index 7880cc4..0f86cd0 100644 --- a/src/providers/embeds/upcloud.ts +++ b/src/providers/embeds/upcloud.ts @@ -4,6 +4,9 @@ import { flags } from '@/entrypoint/utils/targets'; import { makeEmbed } from '@/providers/base'; import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; +const origin = 'https://rabbitstream.net'; +const referer = 'https://rabbitstream.net/'; + const { AES, enc } = crypto; interface StreamRes { @@ -126,6 +129,10 @@ export const upcloudScraper = makeEmbed({ playlist: sources.file, flags: [flags.CORS_ALLOWED], captions, + preferredHeaders: { + Referer: referer, + Origin: origin, + }, }, ], }; diff --git a/src/providers/embeds/vidplay/common.ts b/src/providers/embeds/vidplay/common.ts index 224e3dc..3eb19a9 100644 --- a/src/providers/embeds/vidplay/common.ts +++ b/src/providers/embeds/vidplay/common.ts @@ -2,15 +2,14 @@ import { makeFullUrl } from '@/fetchers/common'; import { decodeData } from '@/providers/sources/vidsrcto/common'; import { EmbedScrapeContext } from '@/utils/context'; -export const vidplayBase = 'https://vidplay.site'; +export const vidplayBase = 'https://vidplay.online'; +export const referer = `${vidplayBase}/`; // This file is based on https://github.com/Ciarands/vidsrc-to-resolver/blob/dffa45e726a4b944cb9af0c9e7630476c93c0213/vidsrc.py#L16 // Full credits to @Ciarands! export const getDecryptionKeys = async (ctx: EmbedScrapeContext): Promise => { - const res = await ctx.fetcher( - 'https://raw.githubusercontent.com/Claudemirovsky/worstsource-keys/keys/keys.json', - ); + const res = await ctx.fetcher('https://raw.githubusercontent.com/Ciarands/vidsrc-keys/main/keys.json'); return JSON.parse(res); }; diff --git a/src/providers/embeds/vidplay/index.ts b/src/providers/embeds/vidplay/index.ts index 3c1f6a2..48af6c1 100644 --- a/src/providers/embeds/vidplay/index.ts +++ b/src/providers/embeds/vidplay/index.ts @@ -1,7 +1,7 @@ import { makeEmbed } from '@/providers/base'; import { Caption, getCaptionTypeFromUrl, labelToLanguageCode } from '@/providers/captions'; -import { getFileUrl } from './common'; +import { getFileUrl, referer } from './common'; import { SubtitleResult, VidplaySourceResponse } from './types'; export const vidplayScraper = makeEmbed({ @@ -46,6 +46,10 @@ export const vidplayScraper = makeEmbed({ playlist: source, flags: [], captions, + preferredHeaders: { + Referer: referer, + Origin: referer, + }, }, ], }; diff --git a/src/providers/embeds/vidsrc.ts b/src/providers/embeds/vidsrc.ts index cd47e21..069c2f6 100644 --- a/src/providers/embeds/vidsrc.ts +++ b/src/providers/embeds/vidsrc.ts @@ -4,6 +4,14 @@ import { makeEmbed } from '@/providers/base'; const hlsURLRegex = /file:"(.*?)"/; const setPassRegex = /var pass_path = "(.*set_pass\.php.*)";/; +function formatHlsB64(data: string): string { + const encodedB64 = data.replace(/\/@#@\/[^=/]+==/g, ''); + if (encodedB64.match(/\/@#@\/[^=/]+==/)) { + return formatHlsB64(encodedB64); + } + return encodedB64; +} + export const vidsrcembedScraper = makeEmbed({ id: 'vidsrcembed', // VidSrc is both a source and an embed host name: 'VidSrc', @@ -15,13 +23,12 @@ export const vidsrcembedScraper = makeEmbed({ }, }); - const match = html - .match(hlsURLRegex)?.[1] - ?.replace(/(\/\/\S+?=)/g, '') - .replace('#2', ''); - if (!match) throw new Error('Unable to find HLS playlist'); - const finalUrl = atob(match); - + // When this eventually breaks see the player js @ pjs_main.js + // If you know what youre doing and are slightly confused about how to reverse this feel free to reach out to ciaran_ds on discord with any queries + let hlsMatch = html.match(hlsURLRegex)?.[1]?.slice(2); + if (!hlsMatch) throw new Error('Unable to find HLS playlist'); + hlsMatch = formatHlsB64(hlsMatch); + const finalUrl = atob(hlsMatch); if (!finalUrl.includes('.m3u8')) throw new Error('Unable to find HLS playlist'); let setPassLink = html.match(setPassRegex)?.[1]; diff --git a/src/providers/sources/lookmovie/index.ts b/src/providers/sources/lookmovie/index.ts index 8611373..6fbe6de 100644 --- a/src/providers/sources/lookmovie/index.ts +++ b/src/providers/sources/lookmovie/index.ts @@ -32,7 +32,7 @@ async function universalScraper(ctx: MovieScrapeContext | ShowScrapeContext): Pr export const lookmovieScraper = makeSourcerer({ id: 'lookmovie', name: 'LookMovie', - rank: 1, + rank: 700, flags: [flags.IP_LOCKED], scrapeShow: universalScraper, scrapeMovie: universalScraper, diff --git a/src/providers/sources/remotestream.ts b/src/providers/sources/remotestream.ts index 8a3090b..fa2efc4 100644 --- a/src/providers/sources/remotestream.ts +++ b/src/providers/sources/remotestream.ts @@ -4,6 +4,9 @@ import { NotFoundError } from '@/utils/errors'; const remotestreamBase = atob('aHR0cHM6Ly9mc2IuOG1ldDNkdGpmcmNxY2hjb25xcGtsd3hzeGIyb2N1bWMuc3RyZWFt'); +const origin = 'https://remotestre.am'; +const referer = 'https://remotestre.am/'; + export const remotestreamScraper = makeSourcerer({ id: 'remotestream', name: 'Remote Stream', @@ -16,9 +19,12 @@ export const remotestreamScraper = makeSourcerer({ const playlistLink = `${remotestreamBase}/Shows/${ctx.media.tmdbId}/${seasonNumber}/${episodeNumber}/${episodeNumber}.m3u8`; ctx.progress(30); - const streamRes = await ctx.fetcher.full(playlistLink, { - method: 'HEAD', + const streamRes = await ctx.proxiedFetcher.full(playlistLink, { + method: 'GET', readHeaders: ['content-type'], + headers: { + Referer: referer, + }, }); if (!streamRes.headers.get('content-type')?.toLowerCase().includes('application/x-mpegurl')) throw new NotFoundError('No watchable item found'); @@ -33,6 +39,10 @@ export const remotestreamScraper = makeSourcerer({ playlist: playlistLink, type: 'hls', flags: [flags.CORS_ALLOWED], + preferredHeaders: { + Referer: referer, + Origin: origin, + }, }, ], }; @@ -41,9 +51,12 @@ export const remotestreamScraper = makeSourcerer({ const playlistLink = `${remotestreamBase}/Movies/${ctx.media.tmdbId}/${ctx.media.tmdbId}.m3u8`; ctx.progress(30); - const streamRes = await ctx.fetcher.full(playlistLink, { - method: 'HEAD', + const streamRes = await ctx.proxiedFetcher.full(playlistLink, { + method: 'GET', readHeaders: ['content-type'], + headers: { + Referer: referer, + }, }); if (!streamRes.headers.get('content-type')?.toLowerCase().includes('application/x-mpegurl')) throw new NotFoundError('No watchable item found'); @@ -58,6 +71,10 @@ export const remotestreamScraper = makeSourcerer({ playlist: playlistLink, type: 'hls', flags: [flags.CORS_ALLOWED], + preferredHeaders: { + Referer: referer, + Origin: origin, + }, }, ], }; diff --git a/src/providers/sources/showbox/index.ts b/src/providers/sources/showbox/index.ts index d6c4887..c8a834a 100644 --- a/src/providers/sources/showbox/index.ts +++ b/src/providers/sources/showbox/index.ts @@ -42,7 +42,7 @@ export const showboxScraper = makeSourcerer({ id: 'showbox', name: 'Showbox', rank: 300, - flags: [flags.CORS_ALLOWED], + flags: [flags.CORS_ALLOWED, flags.CF_BLOCKED], scrapeShow: comboScraper, scrapeMovie: comboScraper, }); diff --git a/vite.config.js b/vite.config.js index e5d4f10..8f6597d 100644 --- a/vite.config.js +++ b/vite.config.js @@ -5,7 +5,9 @@ const dts = require('vite-plugin-dts'); const pkg = require('./package.json'); const fs = require('fs/promises'); -const main = path.resolve(__dirname, 'src/index.ts'); +const shouldTestProviders = process.env.MW_TEST_PROVIDERS === "true" +let tests = ['src/__test__/standard/**/*.test.ts']; +if (shouldTestProviders) tests = ['src/__test__/providers/**/*.test.ts'] module.exports = defineConfig({ plugins: [ @@ -34,10 +36,13 @@ module.exports = defineConfig({ outDir: 'lib', lib: { - entry: main, + entry: path.resolve(__dirname, 'src/index.ts'), name: 'index', fileName: 'index', formats: ['umd', 'es'], }, }, + test: { + include: tests + } });