From 3c292baf2629895f785e78e2532ec0a907b88898 Mon Sep 17 00:00:00 2001 From: Isra Date: Wed, 27 Sep 2023 22:49:22 -0500 Subject: [PATCH 01/22] Add flixhq shows --- package-lock.json | 58 +++++++++++++------ package.json | 5 +- src/dev-cli.ts | 4 +- src/providers/sources/flixhq/index.ts | 19 +++++- src/providers/sources/flixhq/scrape.ts | 58 ++++++++++++++++++- src/providers/sources/flixhq/search.ts | 4 +- .../sources/superstream/sendRequest.ts | 5 +- src/utils/compare.ts | 10 +++- 8 files changed, 129 insertions(+), 34 deletions(-) diff --git a/package-lock.json b/package-lock.json index a9ada68..ab52e74 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,12 +12,13 @@ "cheerio": "^1.0.0-rc.12", "crypto-js": "^4.1.1", "form-data": "^4.0.0", - "nanoid": "^5.0.1", - "node-fetch": "^2.7.0" + "node-fetch": "^2.7.0", + "randombytes": "^2.1.0" }, "devDependencies": { "@types/crypto-js": "^4.1.1", "@types/node-fetch": "^2.6.6", + "@types/randombytes": "^2.0.1", "@types/spinnies": "^0.5.1", "@typescript-eslint/eslint-plugin": "^5.60.0", "@typescript-eslint/parser": "^5.60.0", @@ -942,6 +943,15 @@ "form-data": "^4.0.0" } }, + "node_modules/@types/randombytes": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@types/randombytes/-/randombytes-2.0.1.tgz", + "integrity": "sha512-kWMqPyxpTUTofwbGN47MWddBFiJnWJlfLBdDg2NvmZSKHOmKY9ujVA3PIfBgXcIHTCpsqoQqYudBwanFXzGD9A==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/semver": { "version": "7.5.0", "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.5.0.tgz", @@ -4698,23 +4708,6 @@ "url": "https://github.com/sponsors/raouldeheer" } }, - "node_modules/nanoid": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.1.tgz", - "integrity": "sha512-vWeVtV5Cw68aML/QaZvqN/3QQXc6fBfIieAlu05m7FZW2Dgb+3f0xc0TTxuJW+7u30t7iSDTV/j3kVI0oJqIfQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "bin": { - "nanoid": "bin/nanoid.js" - }, - "engines": { - "node": "^18 || >=20" - } - }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -5258,6 +5251,14 @@ } ] }, + "node_modules/randombytes": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", + "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", + "dependencies": { + "safe-buffer": "^5.1.0" + } + }, "node_modules/react-is": { "version": "17.0.2", "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", @@ -5428,6 +5429,25 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/safe-regex-test": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.0.tgz", diff --git a/package.json b/package.json index 9c8964d..0db14f9 100644 --- a/package.json +++ b/package.json @@ -48,6 +48,7 @@ "devDependencies": { "@types/crypto-js": "^4.1.1", "@types/node-fetch": "^2.6.6", + "@types/randombytes": "^2.0.1", "@types/spinnies": "^0.5.1", "@typescript-eslint/eslint-plugin": "^5.60.0", "@typescript-eslint/parser": "^5.60.0", @@ -76,7 +77,7 @@ "cheerio": "^1.0.0-rc.12", "crypto-js": "^4.1.1", "form-data": "^4.0.0", - "nanoid": "^5.0.1", - "node-fetch": "^2.7.0" + "node-fetch": "^2.7.0", + "randombytes": "^2.1.0" } } diff --git a/src/dev-cli.ts b/src/dev-cli.ts index 2e691da..b2161b3 100644 --- a/src/dev-cli.ts +++ b/src/dev-cli.ts @@ -183,7 +183,7 @@ async function runScraper(providers: ProviderControls, source: MetaOutput, optio id: source.id, }); spinnies.succeed('scrape', { text: 'Done!' }); - console.log(result); + console.log(JSON.stringify(result, null, 2)); } catch (error) { let message = 'Unknown error'; if (error instanceof Error) { @@ -208,7 +208,7 @@ async function runScraper(providers: ProviderControls, source: MetaOutput, optio id: source.id, }); spinnies.succeed('scrape', { text: 'Done!' }); - console.log(result); + console.log(JSON.stringify(result, null, 2)); } catch (error) { let message = 'Unknown error'; if (error instanceof Error) { diff --git a/src/providers/sources/flixhq/index.ts b/src/providers/sources/flixhq/index.ts index fcd504d..c8a519a 100644 --- a/src/providers/sources/flixhq/index.ts +++ b/src/providers/sources/flixhq/index.ts @@ -15,10 +15,27 @@ export const flixhqScraper = makeSourcerer({ const id = await getFlixhqId(ctx, ctx.media); if (!id) throw new NotFoundError('no search results match'); - const sources = await getFlixhqSources(ctx, id); + const sources = await getFlixhqSources(ctx, ctx.media, id); const upcloudStream = sources.find((v) => v.embed.toLowerCase() === 'upcloud'); if (!upcloudStream) throw new NotFoundError('upcloud stream not found for flixhq'); + return { + embeds: [ + { + embedId: upcloudScraper.id, + url: await getFlixhqSourceDetails(ctx, upcloudStream.episodeId), + }, + ], + }; + }, + async scrapeShow(ctx) { + const id = await getFlixhqId(ctx, ctx.media); + if (!id) throw new NotFoundError('no search results match'); + + const sources = await getFlixhqSources(ctx, ctx.media, id); + const upcloudStream = sources.find((v) => v.embed.toLowerCase() === 'server upcloud'); + if (!upcloudStream) throw new NotFoundError('upcloud stream not found for flixhq'); + return { embeds: [ { diff --git a/src/providers/sources/flixhq/scrape.ts b/src/providers/sources/flixhq/scrape.ts index 5c25682..e5707eb 100644 --- a/src/providers/sources/flixhq/scrape.ts +++ b/src/providers/sources/flixhq/scrape.ts @@ -1,16 +1,68 @@ import { load } from 'cheerio'; +import { MovieMedia, ShowMedia } from '@/main/media'; import { flixHqBase } from '@/providers/sources/flixhq/common'; import { ScrapeContext } from '@/utils/context'; +import { NotFoundError } from '@/utils/errors'; -export async function getFlixhqSources(ctx: ScrapeContext, id: string) { - const type = id.split('/')[0]; +export async function getFlixhqSources(ctx: ScrapeContext, media: MovieMedia | ShowMedia, id: string) { const episodeParts = id.split('-'); const episodeId = episodeParts[episodeParts.length - 1]; - const data = await ctx.proxiedFetcher(`/ajax/${type}/episodes/${episodeId}`, { + if (media.type === 'show') { + const seasonsListData = await ctx.proxiedFetcher(`/ajax/season/list/${episodeId}`, { + baseUrl: flixHqBase, + }); + + const seasonsDoc = load(seasonsListData); + const season = seasonsDoc('.dropdown-item') + .toArray() + .find((el) => seasonsDoc(el).text() === `Season ${media.season.number}`)?.attribs['data-id']; + + if (!season) throw new NotFoundError('season not found'); + + const seasonData = await ctx.proxiedFetcher(`/ajax/season/episodes/${season}`, { + baseUrl: flixHqBase, + }); + const seasonDoc = load(seasonData); + const episode = seasonDoc('.nav-item > a') + .toArray() + .map((el) => { + return { + id: seasonDoc(el).attr('data-id'), + title: seasonDoc(el).attr('title'), + }; + }) + .find((e) => e.title?.startsWith(`Eps ${media.episode.number}`))?.id; + + if (!episode) throw new NotFoundError('episode not found'); + + const data = await ctx.proxiedFetcher(`/ajax/episode/servers/${episode}`, { + baseUrl: flixHqBase, + }); + + const doc = load(data); + + const sourceLinks = doc('.nav-item > a') + .toArray() + .map((el) => { + const query = doc(el); + const embedTitle = query.attr('title'); + const linkId = query.attr('data-id'); + if (!embedTitle || !linkId) throw new Error('invalid sources'); + return { + embed: embedTitle, + episodeId: linkId, + }; + }); + + return sourceLinks; + } + + const data = await ctx.proxiedFetcher(`/ajax/movie/episodes/${episodeId}`, { baseUrl: flixHqBase, }); + const doc = load(data); const sourceLinks = doc('.nav-item > a') .toArray() diff --git a/src/providers/sources/flixhq/search.ts b/src/providers/sources/flixhq/search.ts index 569d2d1..cea1b14 100644 --- a/src/providers/sources/flixhq/search.ts +++ b/src/providers/sources/flixhq/search.ts @@ -1,11 +1,11 @@ import { load } from 'cheerio'; -import { MovieMedia } from '@/main/media'; +import { MovieMedia, ShowMedia } from '@/main/media'; import { flixHqBase } from '@/providers/sources/flixhq/common'; import { compareMedia } from '@/utils/compare'; import { ScrapeContext } from '@/utils/context'; -export async function getFlixhqId(ctx: ScrapeContext, media: MovieMedia): Promise { +export async function getFlixhqId(ctx: ScrapeContext, media: MovieMedia | ShowMedia): Promise { const searchResults = await ctx.proxiedFetcher(`/search/${media.title.replaceAll(/[^a-z0-9A-Z]/g, '-')}`, { baseUrl: flixHqBase, }); diff --git a/src/providers/sources/superstream/sendRequest.ts b/src/providers/sources/superstream/sendRequest.ts index ed67a35..7c0122c 100644 --- a/src/providers/sources/superstream/sendRequest.ts +++ b/src/providers/sources/superstream/sendRequest.ts @@ -1,12 +1,11 @@ import CryptoJS from 'crypto-js'; -import { customAlphabet } from 'nanoid'; +import randomBytes from 'randombytes'; import type { ScrapeContext } from '@/utils/context'; import { apiUrls, appId, appKey, key } from './common'; import { encrypt, getVerify } from './crypto'; -const nanoid = customAlphabet('0123456789abcdef', 32); const expiry = () => Math.floor(Date.now() / 1000 + 60 * 60 * 12); export const sendRequest = async (ctx: ScrapeContext, data: object, altApi = false) => { @@ -40,7 +39,7 @@ export const sendRequest = async (ctx: ScrapeContext, data: object, altApi = fal formatted.append('platform', 'android'); formatted.append('version', '129'); formatted.append('medium', 'Website'); - formatted.append('token', nanoid()); + formatted.append('token', randomBytes(16).toString('hex')); const requestUrl = altApi ? apiUrls[1] : apiUrls[0]; diff --git a/src/utils/compare.ts b/src/utils/compare.ts index 8cce7da..ba1b5e4 100644 --- a/src/utils/compare.ts +++ b/src/utils/compare.ts @@ -12,8 +12,14 @@ export function compareTitle(a: string, b: string): boolean { return normalizeTitle(a) === normalizeTitle(b); } -export function compareMedia(media: CommonMedia, title: string, releaseYear?: number): boolean { +export function compareMedia(media: CommonMedia, title: string, releaseYear?: number, compareYear?: boolean): boolean { // if no year is provided, count as if its the correct year - const isSameYear = releaseYear === undefined ? true : media.releaseYear === releaseYear; + let isSameYear: boolean; + if (!compareYear) { + isSameYear = true; + } else { + isSameYear = releaseYear === undefined ? true : media.releaseYear === releaseYear; + } + return compareTitle(media.title, title) && isSameYear; } From e0707019057b3f2b401c34cc2540c84d5840f386 Mon Sep 17 00:00:00 2001 From: Isra Date: Thu, 28 Sep 2023 15:51:39 -0500 Subject: [PATCH 02/22] chore: fix typo --- src/utils/errors.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/errors.ts b/src/utils/errors.ts index d31f7d8..0c83611 100644 --- a/src/utils/errors.ts +++ b/src/utils/errors.ts @@ -1,6 +1,6 @@ export class NotFoundError extends Error { constructor(reason?: string) { - super(`Couldn't found a stream: ${reason ?? 'not found'}`); + super(`Couldn't find a stream: ${reason ?? 'not found'}`); this.name = 'NotFoundError'; } } From 2f289d93d0ae1cbb62becaeaad9c39915b70803e Mon Sep 17 00:00:00 2001 From: Isra Date: Thu, 28 Sep 2023 15:55:02 -0500 Subject: [PATCH 03/22] provider: showbox.media --- src/providers/all.ts | 7 ++- src/providers/embeds/febBox.ts | 58 ++++++++++++++++++++++ src/providers/sources/flixhq/search.ts | 2 +- src/providers/sources/showbox/index.ts | 66 ++++++++++++++++++++++++++ src/utils/compare.ts | 10 +--- 5 files changed, 132 insertions(+), 11 deletions(-) create mode 100644 src/providers/embeds/febBox.ts create mode 100644 src/providers/sources/showbox/index.ts diff --git a/src/providers/all.ts b/src/providers/all.ts index 968b61d..65383c7 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -1,4 +1,5 @@ import { Embed, Sourcerer } from '@/providers/base'; +import { febBoxScraper } from '@/providers/embeds/febBox'; import { mp4uploadScraper } from '@/providers/embeds/mp4upload'; import { streamsbScraper } from '@/providers/embeds/streamsb'; import { upcloudScraper } from '@/providers/embeds/upcloud'; @@ -8,12 +9,14 @@ import { kissAsianScraper } from '@/providers/sources/kissasian/index'; import { remotestreamScraper } from '@/providers/sources/remotestream'; import { superStreamScraper } from '@/providers/sources/superstream/index'; +import { showBoxScraper } from './sources/showbox'; + export function gatherAllSources(): Array { // all sources are gathered here - return [flixhqScraper, remotestreamScraper, kissAsianScraper, superStreamScraper, goMoviesScraper]; + return [flixhqScraper, remotestreamScraper, kissAsianScraper, superStreamScraper, goMoviesScraper, showBoxScraper]; } export function gatherAllEmbeds(): Array { // all embeds are gathered here - return [upcloudScraper, mp4uploadScraper, streamsbScraper]; + return [upcloudScraper, mp4uploadScraper, streamsbScraper, febBoxScraper]; } diff --git a/src/providers/embeds/febBox.ts b/src/providers/embeds/febBox.ts new file mode 100644 index 0000000..ce7e5e9 --- /dev/null +++ b/src/providers/embeds/febBox.ts @@ -0,0 +1,58 @@ +import { flags } from '@/main/targets'; +import { makeEmbed } from '@/providers/base'; +import { NotFoundError } from '@/utils/errors'; + +const febBoxBase = `https://www.febbox.com`; + +export const febBoxScraper = makeEmbed({ + id: 'febBox', + name: 'FebBox', + rank: 160, + async scrape(ctx) { + const shareKey = ctx.url.split('/')[4]; + const streams = await ctx.fetcher<{ + data?: { + file_list?: { + fid?: string; + }[]; + }; + }>('/file/file_share_list', { + baseUrl: febBoxBase, + query: { + share_key: shareKey, + pwd: '', + }, + }); + + const fid = streams?.data?.file_list?.[0]?.fid; + if (!fid) throw new NotFoundError('no result found'); + + const formParams = new URLSearchParams(); + formParams.append('fid', fid); + formParams.append('share_key', shareKey); + + const player = await ctx.fetcher('/file/player', { + baseUrl: febBoxBase, + body: formParams, + method: 'POST', + }); + + const sourcesMatch = player?.match(/var sources = (\[[^\]]+\]);/); + const qualities = sourcesMatch ? JSON.parse(sourcesMatch[0].replace('var sources = ', '').replace(';', '')) : null; + + if (!qualities) throw new NotFoundError('no result found'); + + return { + stream: { + type: 'file', + flags: [flags.NO_CORS], + qualities: { + '360': { + type: 'mp4', + url: qualities[0].file, + }, + }, + }, + }; + }, +}); diff --git a/src/providers/sources/flixhq/search.ts b/src/providers/sources/flixhq/search.ts index cea1b14..90517f6 100644 --- a/src/providers/sources/flixhq/search.ts +++ b/src/providers/sources/flixhq/search.ts @@ -23,7 +23,7 @@ export async function getFlixhqId(ctx: ScrapeContext, media: MovieMedia | ShowMe return { id, title, - year: +year, + year: parseInt(year, 10), }; }); diff --git a/src/providers/sources/showbox/index.ts b/src/providers/sources/showbox/index.ts new file mode 100644 index 0000000..ed8a82c --- /dev/null +++ b/src/providers/sources/showbox/index.ts @@ -0,0 +1,66 @@ +// https://www.febbox.com/file/file_share_list?share_key=${id}&pwd= > data.file_list[0].fid +// https://www.febbox.com/file/player FORM: {hare_key} > "parse js" + +import { load } from 'cheerio'; + +import { flags } from '@/main/targets'; +import { makeSourcerer } from '@/providers/base'; +import { febBoxScraper } from '@/providers/embeds/febBox'; +import { compareMedia } from '@/utils/compare'; +import { NotFoundError } from '@/utils/errors'; + +const showboxBase = `https://www.showbox.media`; + +export const showBoxScraper = makeSourcerer({ + id: 'showBox', + name: 'ShowBox', + rank: 20, + flags: [flags.NO_CORS], + async scrapeMovie(ctx) { + const search = await ctx.fetcher('/search', { + baseUrl: showboxBase, + query: { + keyword: ctx.media.title, + }, + }); + + const searchPage = load(search); + const result = searchPage('.film-name > a') + .toArray() + .map((el) => { + const titleContainer = el.parent?.parent; + if (!titleContainer) return; + const year = searchPage(titleContainer).find('.fdi-item').first().text(); + + return { + title: el.attribs.title, + path: el.attribs.href, + year: !year.includes('SS') ? parseInt(year, 10) : undefined, + }; + }) + .find((v) => v && compareMedia(ctx.media, v.title, v.year ? v.year : undefined)); + + if (!result?.path) throw new NotFoundError('no result found'); + + const febboxResult = await ctx.fetcher<{ + data?: { link?: string }; + }>('/index/share_link', { + baseUrl: showboxBase, + query: { + id: result.path.split('/')[3], + type: '1', + }, + }); + + if (!febboxResult?.data?.link) throw new NotFoundError('no result found'); + + return { + embeds: [ + { + embedId: febBoxScraper.id, + url: febboxResult.data.link, + }, + ], + }; + }, +}); diff --git a/src/utils/compare.ts b/src/utils/compare.ts index ba1b5e4..8cce7da 100644 --- a/src/utils/compare.ts +++ b/src/utils/compare.ts @@ -12,14 +12,8 @@ export function compareTitle(a: string, b: string): boolean { return normalizeTitle(a) === normalizeTitle(b); } -export function compareMedia(media: CommonMedia, title: string, releaseYear?: number, compareYear?: boolean): boolean { +export function compareMedia(media: CommonMedia, title: string, releaseYear?: number): boolean { // if no year is provided, count as if its the correct year - let isSameYear: boolean; - if (!compareYear) { - isSameYear = true; - } else { - isSameYear = releaseYear === undefined ? true : media.releaseYear === releaseYear; - } - + const isSameYear = releaseYear === undefined ? true : media.releaseYear === releaseYear; return compareTitle(media.title, title) && isSameYear; } From 7feaf9256fb3e9e4dffae7b2131088deacb98025 Mon Sep 17 00:00:00 2001 From: Isra Date: Fri, 29 Sep 2023 22:44:59 -0500 Subject: [PATCH 04/22] febbox use actual qualities --- src/providers/embeds/febBox.ts | 21 ++++++++++++------- src/providers/sources/showbox/index.ts | 3 --- .../sources/superstream/getStreamQualities.ts | 2 +- src/providers/sources/superstream/index.ts | 2 -- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/providers/embeds/febBox.ts b/src/providers/embeds/febBox.ts index ce7e5e9..daea6fd 100644 --- a/src/providers/embeds/febBox.ts +++ b/src/providers/embeds/febBox.ts @@ -1,9 +1,12 @@ import { flags } from '@/main/targets'; import { makeEmbed } from '@/providers/base'; +import { StreamFile } from '@/providers/streams'; import { NotFoundError } from '@/utils/errors'; const febBoxBase = `https://www.febbox.com`; +const allowedQualities = ['360', '480', '720', '1080']; + export const febBoxScraper = makeEmbed({ id: 'febBox', name: 'FebBox', @@ -40,18 +43,22 @@ export const febBoxScraper = makeEmbed({ const sourcesMatch = player?.match(/var sources = (\[[^\]]+\]);/); const qualities = sourcesMatch ? JSON.parse(sourcesMatch[0].replace('var sources = ', '').replace(';', '')) : null; - if (!qualities) throw new NotFoundError('no result found'); + const embedQualities: Record = {}; + + qualities.forEach((quality: { file: string; label: string }) => { + if (allowedQualities.includes(quality.label.replace('P', ''))) { + embedQualities[quality.label.replace('p', '')] = { + type: 'mp4', + url: quality.file, + }; + } + }); return { stream: { type: 'file', flags: [flags.NO_CORS], - qualities: { - '360': { - type: 'mp4', - url: qualities[0].file, - }, - }, + qualities: embedQualities, }, }; }, diff --git a/src/providers/sources/showbox/index.ts b/src/providers/sources/showbox/index.ts index ed8a82c..81fe798 100644 --- a/src/providers/sources/showbox/index.ts +++ b/src/providers/sources/showbox/index.ts @@ -1,6 +1,3 @@ -// https://www.febbox.com/file/file_share_list?share_key=${id}&pwd= > data.file_list[0].fid -// https://www.febbox.com/file/player FORM: {hare_key} > "parse js" - import { load } from 'cheerio'; import { flags } from '@/main/targets'; diff --git a/src/providers/sources/superstream/getStreamQualities.ts b/src/providers/sources/superstream/getStreamQualities.ts index 6fe55a3..8b04f2d 100644 --- a/src/providers/sources/superstream/getStreamQualities.ts +++ b/src/providers/sources/superstream/getStreamQualities.ts @@ -3,7 +3,7 @@ import { ScrapeContext } from '@/utils/context'; import { sendRequest } from './sendRequest'; -import { allowedQualities } from '.'; +const allowedQualities = ['360', '480', '720', '1080']; export async function getStreamQualities(ctx: ScrapeContext, apiQuery: object) { const mediaRes: { list: { path: string; real_quality: string }[] } = (await sendRequest(ctx, apiQuery)).data; diff --git a/src/providers/sources/superstream/index.ts b/src/providers/sources/superstream/index.ts index 45f64b2..78972ea 100644 --- a/src/providers/sources/superstream/index.ts +++ b/src/providers/sources/superstream/index.ts @@ -6,8 +6,6 @@ import { NotFoundError } from '@/utils/errors'; import { getStreamQualities } from './getStreamQualities'; import { sendRequest } from './sendRequest'; -export const allowedQualities = ['360', '480', '720', '1080']; - export const superStreamScraper = makeSourcerer({ id: 'superstream', name: 'Superstream', From 791caa88fde37bfae7fb9a57a86256fc595a11a9 Mon Sep 17 00:00:00 2001 From: Isra Date: Sun, 1 Oct 2023 15:18:14 -0500 Subject: [PATCH 05/22] refactor: use util.inspect instead of a stringify --- src/dev-cli.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/dev-cli.ts b/src/dev-cli.ts index b2161b3..fc8f2f3 100644 --- a/src/dev-cli.ts +++ b/src/dev-cli.ts @@ -1,3 +1,5 @@ +import util from 'node:util'; + // eslint-disable-next-line import/no-extraneous-dependencies import { program } from 'commander'; // eslint-disable-next-line import/no-extraneous-dependencies @@ -47,6 +49,10 @@ if (!TMDB_API_KEY?.trim()) { throw new Error('Missing MOVIE_WEB_TMDB_API_KEY environment variable'); } +function showOutput(object: object) { + console.log(util.inspect(object, { showHidden: false, depth: null, colors: true })); +} + function getAllSources() { // * The only way to get a list of all sources is to // * create all these things. Maybe this should change From c9bd98517fc9a79d3689f96ce38dd3bc82d70861 Mon Sep 17 00:00:00 2001 From: Isra Date: Tue, 3 Oct 2023 12:23:23 -0500 Subject: [PATCH 06/22] fix: emoty string files for superstream --- src/providers/embeds/febBox.ts | 1 + src/providers/sources/superstream/getStreamQualities.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/providers/embeds/febBox.ts b/src/providers/embeds/febBox.ts index daea6fd..276e3dc 100644 --- a/src/providers/embeds/febBox.ts +++ b/src/providers/embeds/febBox.ts @@ -47,6 +47,7 @@ export const febBoxScraper = makeEmbed({ qualities.forEach((quality: { file: string; label: string }) => { if (allowedQualities.includes(quality.label.replace('P', ''))) { + if (!quality.file) return; embedQualities[quality.label.replace('p', '')] = { type: 'mp4', url: quality.file, diff --git a/src/providers/sources/superstream/getStreamQualities.ts b/src/providers/sources/superstream/getStreamQualities.ts index 8b04f2d..f968a18 100644 --- a/src/providers/sources/superstream/getStreamQualities.ts +++ b/src/providers/sources/superstream/getStreamQualities.ts @@ -20,7 +20,7 @@ export async function getStreamQualities(ctx: ScrapeContext, apiQuery: object) { allowedQualities.forEach((quality) => { const foundQuality = qualityMap.find((q) => q.quality === quality); - if (foundQuality) { + if (foundQuality && foundQuality.url) { qualities[quality] = { type: 'mp4', url: foundQuality.url, From 8baeb90bee0175de90d7cb817f5f6682fe4d44a7 Mon Sep 17 00:00:00 2001 From: Isra Date: Tue, 3 Oct 2023 16:44:34 -0500 Subject: [PATCH 07/22] fix: refactor showOutput to logDeepObject --- src/dev-cli.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dev-cli.ts b/src/dev-cli.ts index fc8f2f3..f496679 100644 --- a/src/dev-cli.ts +++ b/src/dev-cli.ts @@ -49,7 +49,7 @@ if (!TMDB_API_KEY?.trim()) { throw new Error('Missing MOVIE_WEB_TMDB_API_KEY environment variable'); } -function showOutput(object: object) { +function logDeepObject(object: Record) { console.log(util.inspect(object, { showHidden: false, depth: null, colors: true })); } @@ -189,7 +189,7 @@ async function runScraper(providers: ProviderControls, source: MetaOutput, optio id: source.id, }); spinnies.succeed('scrape', { text: 'Done!' }); - console.log(JSON.stringify(result, null, 2)); + console.log(logDeepObject(result)); } catch (error) { let message = 'Unknown error'; if (error instanceof Error) { @@ -214,7 +214,7 @@ async function runScraper(providers: ProviderControls, source: MetaOutput, optio id: source.id, }); spinnies.succeed('scrape', { text: 'Done!' }); - console.log(JSON.stringify(result, null, 2)); + console.log(logDeepObject(result)); } catch (error) { let message = 'Unknown error'; if (error instanceof Error) { From f82457ed62de692a6854951f01eb182b7ca312bf Mon Sep 17 00:00:00 2001 From: Isra Date: Tue, 3 Oct 2023 16:46:27 -0500 Subject: [PATCH 08/22] fix: use proxied fetcher --- src/providers/embeds/febBox.ts | 4 ++-- src/providers/sources/showbox/index.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/providers/embeds/febBox.ts b/src/providers/embeds/febBox.ts index 276e3dc..ba2a26b 100644 --- a/src/providers/embeds/febBox.ts +++ b/src/providers/embeds/febBox.ts @@ -13,7 +13,7 @@ export const febBoxScraper = makeEmbed({ rank: 160, async scrape(ctx) { const shareKey = ctx.url.split('/')[4]; - const streams = await ctx.fetcher<{ + const streams = await ctx.proxiedFetcher<{ data?: { file_list?: { fid?: string; @@ -34,7 +34,7 @@ export const febBoxScraper = makeEmbed({ formParams.append('fid', fid); formParams.append('share_key', shareKey); - const player = await ctx.fetcher('/file/player', { + const player = await ctx.proxiedFetcher('/file/player', { baseUrl: febBoxBase, body: formParams, method: 'POST', diff --git a/src/providers/sources/showbox/index.ts b/src/providers/sources/showbox/index.ts index 81fe798..1534068 100644 --- a/src/providers/sources/showbox/index.ts +++ b/src/providers/sources/showbox/index.ts @@ -14,7 +14,7 @@ export const showBoxScraper = makeSourcerer({ rank: 20, flags: [flags.NO_CORS], async scrapeMovie(ctx) { - const search = await ctx.fetcher('/search', { + const search = await ctx.proxiedFetcher('/search', { baseUrl: showboxBase, query: { keyword: ctx.media.title, @@ -39,7 +39,7 @@ export const showBoxScraper = makeSourcerer({ if (!result?.path) throw new NotFoundError('no result found'); - const febboxResult = await ctx.fetcher<{ + const febboxResult = await ctx.proxiedFetcher<{ data?: { link?: string }; }>('/index/share_link', { baseUrl: showboxBase, From c3de70bbc05dd4935ab32ee0c136838290111602 Mon Sep 17 00:00:00 2001 From: Isra Date: Tue, 3 Oct 2023 16:48:23 -0500 Subject: [PATCH 09/22] fix: seperate function --- src/providers/sources/flixhq/scrape.ts | 105 +++++++++++++------------ 1 file changed, 54 insertions(+), 51 deletions(-) diff --git a/src/providers/sources/flixhq/scrape.ts b/src/providers/sources/flixhq/scrape.ts index e5707eb..c4a5955 100644 --- a/src/providers/sources/flixhq/scrape.ts +++ b/src/providers/sources/flixhq/scrape.ts @@ -5,60 +5,10 @@ import { flixHqBase } from '@/providers/sources/flixhq/common'; import { ScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; -export async function getFlixhqSources(ctx: ScrapeContext, media: MovieMedia | ShowMedia, id: string) { +export async function getFlixhqMovieSources(ctx: ScrapeContext, media: MovieMedia, id: string) { const episodeParts = id.split('-'); const episodeId = episodeParts[episodeParts.length - 1]; - if (media.type === 'show') { - const seasonsListData = await ctx.proxiedFetcher(`/ajax/season/list/${episodeId}`, { - baseUrl: flixHqBase, - }); - - const seasonsDoc = load(seasonsListData); - const season = seasonsDoc('.dropdown-item') - .toArray() - .find((el) => seasonsDoc(el).text() === `Season ${media.season.number}`)?.attribs['data-id']; - - if (!season) throw new NotFoundError('season not found'); - - const seasonData = await ctx.proxiedFetcher(`/ajax/season/episodes/${season}`, { - baseUrl: flixHqBase, - }); - const seasonDoc = load(seasonData); - const episode = seasonDoc('.nav-item > a') - .toArray() - .map((el) => { - return { - id: seasonDoc(el).attr('data-id'), - title: seasonDoc(el).attr('title'), - }; - }) - .find((e) => e.title?.startsWith(`Eps ${media.episode.number}`))?.id; - - if (!episode) throw new NotFoundError('episode not found'); - - const data = await ctx.proxiedFetcher(`/ajax/episode/servers/${episode}`, { - baseUrl: flixHqBase, - }); - - const doc = load(data); - - const sourceLinks = doc('.nav-item > a') - .toArray() - .map((el) => { - const query = doc(el); - const embedTitle = query.attr('title'); - const linkId = query.attr('data-id'); - if (!embedTitle || !linkId) throw new Error('invalid sources'); - return { - embed: embedTitle, - episodeId: linkId, - }; - }); - - return sourceLinks; - } - const data = await ctx.proxiedFetcher(`/ajax/movie/episodes/${episodeId}`, { baseUrl: flixHqBase, }); @@ -87,3 +37,56 @@ export async function getFlixhqSourceDetails(ctx: ScrapeContext, sourceId: strin return jsonData.link; } + +export async function getFlixhqShowSources(ctx: ScrapeContext, media: ShowMedia, id: string) { + const episodeParts = id.split('-'); + const episodeId = episodeParts[episodeParts.length - 1]; + + const seasonsListData = await ctx.proxiedFetcher(`/ajax/season/list/${episodeId}`, { + baseUrl: flixHqBase, + }); + + const seasonsDoc = load(seasonsListData); + const season = seasonsDoc('.dropdown-item') + .toArray() + .find((el) => seasonsDoc(el).text() === `Season ${media.season.number}`)?.attribs['data-id']; + + if (!season) throw new NotFoundError('season not found'); + + const seasonData = await ctx.proxiedFetcher(`/ajax/season/episodes/${season}`, { + baseUrl: flixHqBase, + }); + const seasonDoc = load(seasonData); + const episode = seasonDoc('.nav-item > a') + .toArray() + .map((el) => { + return { + id: seasonDoc(el).attr('data-id'), + title: seasonDoc(el).attr('title'), + }; + }) + .find((e) => e.title?.startsWith(`Eps ${media.episode.number}`))?.id; + + if (!episode) throw new NotFoundError('episode not found'); + + const data = await ctx.proxiedFetcher(`/ajax/episode/servers/${episode}`, { + baseUrl: flixHqBase, + }); + + const doc = load(data); + + const sourceLinks = doc('.nav-item > a') + .toArray() + .map((el) => { + const query = doc(el); + const embedTitle = query.attr('title'); + const linkId = query.attr('data-id'); + if (!embedTitle || !linkId) throw new Error('invalid sources'); + return { + embed: embedTitle, + episodeId: linkId, + }; + }); + + return sourceLinks; +} From 11af17b19bf60295b40d4df23d5c7d2750b65f3a Mon Sep 17 00:00:00 2001 From: Isra Date: Tue, 3 Oct 2023 16:49:39 -0500 Subject: [PATCH 10/22] fix: correct order of functions --- src/providers/sources/flixhq/scrape.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/providers/sources/flixhq/scrape.ts b/src/providers/sources/flixhq/scrape.ts index c4a5955..a73916c 100644 --- a/src/providers/sources/flixhq/scrape.ts +++ b/src/providers/sources/flixhq/scrape.ts @@ -5,6 +5,14 @@ import { flixHqBase } from '@/providers/sources/flixhq/common'; import { ScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; +export async function getFlixhqSourceDetails(ctx: ScrapeContext, sourceId: string): Promise { + const jsonData = await ctx.proxiedFetcher>(`/ajax/sources/${sourceId}`, { + baseUrl: flixHqBase, + }); + + return jsonData.link; +} + export async function getFlixhqMovieSources(ctx: ScrapeContext, media: MovieMedia, id: string) { const episodeParts = id.split('-'); const episodeId = episodeParts[episodeParts.length - 1]; @@ -30,14 +38,6 @@ export async function getFlixhqMovieSources(ctx: ScrapeContext, media: MovieMedi return sourceLinks; } -export async function getFlixhqSourceDetails(ctx: ScrapeContext, sourceId: string): Promise { - const jsonData = await ctx.proxiedFetcher>(`/ajax/sources/${sourceId}`, { - baseUrl: flixHqBase, - }); - - return jsonData.link; -} - export async function getFlixhqShowSources(ctx: ScrapeContext, media: ShowMedia, id: string) { const episodeParts = id.split('-'); const episodeId = episodeParts[episodeParts.length - 1]; From 856a8fcac1fcb37856729ff5f3f10b417284672a Mon Sep 17 00:00:00 2001 From: Isra Date: Tue, 3 Oct 2023 16:50:54 -0500 Subject: [PATCH 11/22] fix: use updated imports --- src/providers/sources/flixhq/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/providers/sources/flixhq/index.ts b/src/providers/sources/flixhq/index.ts index c8a519a..e5cae5b 100644 --- a/src/providers/sources/flixhq/index.ts +++ b/src/providers/sources/flixhq/index.ts @@ -1,7 +1,7 @@ import { flags } from '@/main/targets'; import { makeSourcerer } from '@/providers/base'; import { upcloudScraper } from '@/providers/embeds/upcloud'; -import { getFlixhqSourceDetails, getFlixhqSources } from '@/providers/sources/flixhq/scrape'; +import { getFlixhqMovieSources, getFlixhqShowSources, getFlixhqSourceDetails } from '@/providers/sources/flixhq/scrape'; import { getFlixhqId } from '@/providers/sources/flixhq/search'; import { NotFoundError } from '@/utils/errors'; @@ -15,7 +15,7 @@ export const flixhqScraper = makeSourcerer({ const id = await getFlixhqId(ctx, ctx.media); if (!id) throw new NotFoundError('no search results match'); - const sources = await getFlixhqSources(ctx, ctx.media, id); + const sources = await getFlixhqMovieSources(ctx, ctx.media, id); const upcloudStream = sources.find((v) => v.embed.toLowerCase() === 'upcloud'); if (!upcloudStream) throw new NotFoundError('upcloud stream not found for flixhq'); @@ -32,7 +32,7 @@ export const flixhqScraper = makeSourcerer({ const id = await getFlixhqId(ctx, ctx.media); if (!id) throw new NotFoundError('no search results match'); - const sources = await getFlixhqSources(ctx, ctx.media, id); + const sources = await getFlixhqShowSources(ctx, ctx.media, id); const upcloudStream = sources.find((v) => v.embed.toLowerCase() === 'server upcloud'); if (!upcloudStream) throw new NotFoundError('upcloud stream not found for flixhq'); From 6879dcdbe7b38ce4890f14b5edd89c789f25d348 Mon Sep 17 00:00:00 2001 From: Isra Date: Tue, 3 Oct 2023 17:33:19 -0500 Subject: [PATCH 12/22] fix: no double log --- src/dev-cli.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dev-cli.ts b/src/dev-cli.ts index f496679..45b2405 100644 --- a/src/dev-cli.ts +++ b/src/dev-cli.ts @@ -189,7 +189,7 @@ async function runScraper(providers: ProviderControls, source: MetaOutput, optio id: source.id, }); spinnies.succeed('scrape', { text: 'Done!' }); - console.log(logDeepObject(result)); + logDeepObject(result); } catch (error) { let message = 'Unknown error'; if (error instanceof Error) { @@ -214,7 +214,7 @@ async function runScraper(providers: ProviderControls, source: MetaOutput, optio id: source.id, }); spinnies.succeed('scrape', { text: 'Done!' }); - console.log(logDeepObject(result)); + logDeepObject(result); } catch (error) { let message = 'Unknown error'; if (error instanceof Error) { From 64b493eaf2dcbb702e15eba743fdcdf6598b0c21 Mon Sep 17 00:00:00 2001 From: Isra Date: Tue, 3 Oct 2023 17:48:40 -0500 Subject: [PATCH 13/22] Fix linter errors --- package-lock.json | 25 ++++++++++++++++++++++--- package.json | 3 ++- src/providers/all.ts | 10 +++++++++- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1a88651..f8d6ab0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,19 +1,20 @@ { "name": "@movie-web/providers", - "version": "1.0.0", + "version": "1.0.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@movie-web/providers", - "version": "1.0.0", + "version": "1.0.1", "license": "MIT", "dependencies": { "cheerio": "^1.0.0-rc.12", "crypto-js": "^4.1.1", "form-data": "^4.0.0", "node-fetch": "^2.7.0", - "randombytes": "^2.1.0" + "randombytes": "^2.1.0", + "unpacker": "^1.0.1" }, "devDependencies": { "@types/crypto-js": "^4.1.1", @@ -4709,6 +4710,24 @@ "url": "https://github.com/sponsors/raouldeheer" } }, + "node_modules/nanoid": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", + "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", diff --git a/package.json b/package.json index 9308bab..9607c1a 100644 --- a/package.json +++ b/package.json @@ -80,6 +80,7 @@ "crypto-js": "^4.1.1", "form-data": "^4.0.0", "node-fetch": "^2.7.0", - "randombytes": "^2.1.0" + "randombytes": "^2.1.0", + "unpacker": "^1.0.1" } } diff --git a/src/providers/all.ts b/src/providers/all.ts index 22a9f0c..49ff4f2 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -16,7 +16,15 @@ import { showBoxScraper } from './sources/showbox'; export function gatherAllSources(): Array { // all sources are gathered here - return [flixhqScraper, remotestreamScraper, kissAsianScraper, superStreamScraper, goMoviesScraper, zoechipScraper, showBoxScraper]; + return [ + flixhqScraper, + remotestreamScraper, + kissAsianScraper, + superStreamScraper, + goMoviesScraper, + zoechipScraper, + showBoxScraper, + ]; } export function gatherAllEmbeds(): Array { From 35cfa4747e955042f4241ba5b1da52da5eb9020a Mon Sep 17 00:00:00 2001 From: Isra Date: Tue, 3 Oct 2023 17:51:08 -0500 Subject: [PATCH 14/22] Fix no-extraneous-dependencies --- src/dev-cli.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dev-cli.ts b/src/dev-cli.ts index 30b435e..54d663a 100644 --- a/src/dev-cli.ts +++ b/src/dev-cli.ts @@ -1,6 +1,7 @@ +/* eslint import/no-extraneous-dependencies: ["error", {"devDependencies": true}] */ + import util from 'node:util'; -// eslint-disable-next-line import/no-extraneous-dependencies import { program } from 'commander'; import dotenv from 'dotenv'; import { prompt } from 'enquirer'; From 6ecc866fed86591369408713964c13c41cbc1440 Mon Sep 17 00:00:00 2001 From: Isra Date: Tue, 3 Oct 2023 17:54:53 -0500 Subject: [PATCH 15/22] Reintroduce nanoid --- package-lock.json | 58 ++++++++----------- package.json | 2 +- .../sources/superstream/sendRequest.ts | 5 +- 3 files changed, 28 insertions(+), 37 deletions(-) diff --git a/package-lock.json b/package-lock.json index f8d6ab0..6f1caf8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,8 +12,8 @@ "cheerio": "^1.0.0-rc.12", "crypto-js": "^4.1.1", "form-data": "^4.0.0", + "nanoid": "^5.0.1", "node-fetch": "^2.7.0", - "randombytes": "^2.1.0", "unpacker": "^1.0.1" }, "devDependencies": { @@ -4711,10 +4711,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", - "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", - "dev": true, + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.1.tgz", + "integrity": "sha512-vWeVtV5Cw68aML/QaZvqN/3QQXc6fBfIieAlu05m7FZW2Dgb+3f0xc0TTxuJW+7u30t7iSDTV/j3kVI0oJqIfQ==", "funding": [ { "type": "github", @@ -4722,10 +4721,10 @@ } ], "bin": { - "nanoid": "bin/nanoid.cjs" + "nanoid": "bin/nanoid.js" }, "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + "node": "^18 || >=20" } }, "node_modules/natural-compare": { @@ -5144,6 +5143,24 @@ "node": "^10 || ^12 || >=14" } }, + "node_modules/postcss/node_modules/nanoid": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", + "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -5257,14 +5274,6 @@ } ] }, - "node_modules/randombytes": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", - "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==", - "dependencies": { - "safe-buffer": "^5.1.0" - } - }, "node_modules/react-is": { "version": "17.0.2", "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz", @@ -5435,25 +5444,6 @@ "queue-microtask": "^1.2.2" } }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/safe-regex-test": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/safe-regex-test/-/safe-regex-test-1.0.0.tgz", diff --git a/package.json b/package.json index 9607c1a..142aa64 100644 --- a/package.json +++ b/package.json @@ -79,8 +79,8 @@ "cheerio": "^1.0.0-rc.12", "crypto-js": "^4.1.1", "form-data": "^4.0.0", + "nanoid": "^5.0.1", "node-fetch": "^2.7.0", - "randombytes": "^2.1.0", "unpacker": "^1.0.1" } } diff --git a/src/providers/sources/superstream/sendRequest.ts b/src/providers/sources/superstream/sendRequest.ts index 7c0122c..2f77767 100644 --- a/src/providers/sources/superstream/sendRequest.ts +++ b/src/providers/sources/superstream/sendRequest.ts @@ -1,11 +1,12 @@ import CryptoJS from 'crypto-js'; -import randomBytes from 'randombytes'; +import { customAlphabet } from 'nanoid'; import type { ScrapeContext } from '@/utils/context'; import { apiUrls, appId, appKey, key } from './common'; import { encrypt, getVerify } from './crypto'; +const randomId = customAlphabet('1234567890abcdef'); const expiry = () => Math.floor(Date.now() / 1000 + 60 * 60 * 12); export const sendRequest = async (ctx: ScrapeContext, data: object, altApi = false) => { @@ -39,7 +40,7 @@ export const sendRequest = async (ctx: ScrapeContext, data: object, altApi = fal formatted.append('platform', 'android'); formatted.append('version', '129'); formatted.append('medium', 'Website'); - formatted.append('token', randomBytes(16).toString('hex')); + formatted.append('token', randomId(32)); const requestUrl = altApi ? apiUrls[1] : apiUrls[0]; From aaa01a629be1a935e0e9abd44d237d0722ff1be3 Mon Sep 17 00:00:00 2001 From: Isra Date: Tue, 3 Oct 2023 17:57:52 -0500 Subject: [PATCH 16/22] Use correct nanoid version --- package-lock.json | 30 ++++++------------------------ package.json | 2 +- 2 files changed, 7 insertions(+), 25 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6f1caf8..40a870f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,7 @@ "cheerio": "^1.0.0-rc.12", "crypto-js": "^4.1.1", "form-data": "^4.0.0", - "nanoid": "^5.0.1", + "nanoid": "^3.3.6", "node-fetch": "^2.7.0", "unpacker": "^1.0.1" }, @@ -4711,9 +4711,9 @@ } }, "node_modules/nanoid": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.0.1.tgz", - "integrity": "sha512-vWeVtV5Cw68aML/QaZvqN/3QQXc6fBfIieAlu05m7FZW2Dgb+3f0xc0TTxuJW+7u30t7iSDTV/j3kVI0oJqIfQ==", + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", + "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", "funding": [ { "type": "github", @@ -4721,10 +4721,10 @@ } ], "bin": { - "nanoid": "bin/nanoid.js" + "nanoid": "bin/nanoid.cjs" }, "engines": { - "node": "^18 || >=20" + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, "node_modules/natural-compare": { @@ -5143,24 +5143,6 @@ "node": "^10 || ^12 || >=14" } }, - "node_modules/postcss/node_modules/nanoid": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", - "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", diff --git a/package.json b/package.json index 142aa64..04d1619 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,7 @@ "cheerio": "^1.0.0-rc.12", "crypto-js": "^4.1.1", "form-data": "^4.0.0", - "nanoid": "^5.0.1", + "nanoid": "^3.3.6", "node-fetch": "^2.7.0", "unpacker": "^1.0.1" } From 6f9c0517b82b793a7502c75549a30127da2afec9 Mon Sep 17 00:00:00 2001 From: Isra Date: Tue, 3 Oct 2023 19:58:54 -0500 Subject: [PATCH 17/22] fix: Exclude capital 'P' from qualities --- src/providers/embeds/febBox.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/providers/embeds/febBox.ts b/src/providers/embeds/febBox.ts index ba2a26b..aa0a21c 100644 --- a/src/providers/embeds/febBox.ts +++ b/src/providers/embeds/febBox.ts @@ -48,7 +48,7 @@ export const febBoxScraper = makeEmbed({ qualities.forEach((quality: { file: string; label: string }) => { if (allowedQualities.includes(quality.label.replace('P', ''))) { if (!quality.file) return; - embedQualities[quality.label.replace('p', '')] = { + embedQualities[quality.label.replace('P', '')] = { type: 'mp4', url: quality.file, }; From 9d204c381fff28c493c8baf6b79dd3bcc56371bd Mon Sep 17 00:00:00 2001 From: mrjvs Date: Thu, 26 Oct 2023 20:52:07 +0200 Subject: [PATCH 18/22] Fix febbox --- src/providers/embeds/febBox.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/providers/embeds/febBox.ts b/src/providers/embeds/febBox.ts index aa0a21c..06a3414 100644 --- a/src/providers/embeds/febBox.ts +++ b/src/providers/embeds/febBox.ts @@ -8,7 +8,7 @@ const febBoxBase = `https://www.febbox.com`; const allowedQualities = ['360', '480', '720', '1080']; export const febBoxScraper = makeEmbed({ - id: 'febBox', + id: 'febbox', name: 'FebBox', rank: 160, async scrape(ctx) { @@ -20,6 +20,9 @@ export const febBoxScraper = makeEmbed({ }[]; }; }>('/file/file_share_list', { + headers: { + 'accept-language': 'en', // without this header, the request is marked as a webscraper + }, baseUrl: febBoxBase, query: { share_key: shareKey, @@ -38,6 +41,9 @@ export const febBoxScraper = makeEmbed({ baseUrl: febBoxBase, body: formParams, method: 'POST', + headers: { + 'accept-language': 'en', // without this header, the request is marked as a webscraper + }, }); const sourcesMatch = player?.match(/var sources = (\[[^\]]+\]);/); @@ -46,9 +52,10 @@ export const febBoxScraper = makeEmbed({ const embedQualities: Record = {}; qualities.forEach((quality: { file: string; label: string }) => { - if (allowedQualities.includes(quality.label.replace('P', ''))) { + const normalizedLabel = quality.label.toLowerCase().replace('p', ''); + if (allowedQualities.includes(normalizedLabel)) { if (!quality.file) return; - embedQualities[quality.label.replace('P', '')] = { + embedQualities[normalizedLabel] = { type: 'mp4', url: quality.file, }; From bec7c07881c8e85c56e308d8eb01b5b5b0d21e7c Mon Sep 17 00:00:00 2001 From: mrjvs Date: Thu, 26 Oct 2023 21:07:11 +0200 Subject: [PATCH 19/22] check validity of stream before returning --- src/main/individualRunner.ts | 5 ++++- src/main/runner.ts | 4 ++++ src/utils/valid.ts | 17 +++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 src/utils/valid.ts diff --git a/src/main/individualRunner.ts b/src/main/individualRunner.ts index 957edd0..ac563ea 100644 --- a/src/main/individualRunner.ts +++ b/src/main/individualRunner.ts @@ -6,6 +6,7 @@ import { EmbedOutput, SourcererOutput } from '@/providers/base'; import { ProviderList } from '@/providers/get'; import { ScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; +import { isValidStream } from '@/utils/valid'; export type IndividualSourceRunnerOptions = { features: FeatureMap; @@ -50,7 +51,7 @@ export async function scrapeInvidualSource( }); // stream doesn't satisfy the feature flags, so gets removed in output - if (output?.stream && !flagsAllowedInFeatures(ops.features, output.stream.flags)) { + if (output?.stream && (!isValidStream(output.stream) || !flagsAllowedInFeatures(ops.features, output.stream.flags))) { output.stream = undefined; } @@ -87,7 +88,9 @@ export async function scrapeIndividualEmbed( }, }); + if (!isValidStream(output.stream)) throw new NotFoundError('stream is incomplete'); if (!flagsAllowedInFeatures(ops.features, output.stream.flags)) throw new NotFoundError("stream doesn't satisfy target feature flags"); + return output; } diff --git a/src/main/runner.ts b/src/main/runner.ts index 0ecaee1..866ffb9 100644 --- a/src/main/runner.ts +++ b/src/main/runner.ts @@ -8,6 +8,7 @@ import { Stream } from '@/providers/streams'; import { ScrapeContext } from '@/utils/context'; import { NotFoundError } from '@/utils/errors'; import { reorderOnIdList } from '@/utils/list'; +import { isValidStream } from '@/utils/valid'; export type RunOutput = { sourceId: string; @@ -79,6 +80,9 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt ...contextBase, media: ops.media, }); + if (!isValidStream(output?.stream)) { + throw new NotFoundError('stream is incomplete'); + } if (output?.stream && !flagsAllowedInFeatures(ops.features, output.stream.flags)) { throw new NotFoundError("stream doesn't satisfy target feature flags"); } diff --git a/src/utils/valid.ts b/src/utils/valid.ts new file mode 100644 index 0000000..62347c3 --- /dev/null +++ b/src/utils/valid.ts @@ -0,0 +1,17 @@ +import { Stream } from '@/providers/streams'; + +export function isValidStream(stream: Stream | undefined): boolean { + if (!stream) return false; + if (stream.type === 'hls') { + if (!stream.playlist) return false; + return true; + } + if (stream.type === 'file') { + const validQualities = Object.values(stream.qualities).filter((v) => v.url.length > 0); + if (validQualities.length === 0) return false; + return true; + } + + // unknown file type + return false; +} From 8647d08e2269c29fd86097dc9d289777f518d792 Mon Sep 17 00:00:00 2001 From: mrjvs Date: Thu, 26 Oct 2023 21:17:44 +0200 Subject: [PATCH 20/22] fix security in dependencies --- .docs/package-lock.json | 70 ++++++++++++++++++++--------------------- package-lock.json | 12 +++---- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/.docs/package-lock.json b/.docs/package-lock.json index f465fb7..315b381 100644 --- a/.docs/package-lock.json +++ b/.docs/package-lock.json @@ -1,11 +1,11 @@ { - "name": "docus-starter", + "name": "providers-docs", "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "docus-starter", + "name": "providers-docs", "version": "0.1.0", "devDependencies": { "@nuxt-themes/docus": "^1.13.1", @@ -157,12 +157,12 @@ } }, "node_modules/@babel/generator": { - "version": "7.22.15", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.22.15.tgz", - "integrity": "sha512-Zu9oWARBqeVOW0dZOjXc3JObrzuqothQ3y/n1kUtrjCoCPLkXUwMvOo/F/TCfoHMbWIFlWwpZtkZVb9ga4U2pA==", + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.0.tgz", + "integrity": "sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g==", "dev": true, "dependencies": { - "@babel/types": "^7.22.15", + "@babel/types": "^7.23.0", "@jridgewell/gen-mapping": "^0.3.2", "@jridgewell/trace-mapping": "^0.3.17", "jsesc": "^2.5.1" @@ -241,22 +241,22 @@ } }, "node_modules/@babel/helper-environment-visitor": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.5.tgz", - "integrity": "sha512-XGmhECfVA/5sAt+H+xpSg0mfrHq6FzNr9Oxh7PSEBBRUb/mL7Kz3NICXb194rCqAEdxkhPT1a88teizAFyvk8Q==", + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz", + "integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==", "dev": true, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-function-name": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.22.5.tgz", - "integrity": "sha512-wtHSq6jMRE3uF2otvfuD3DIvVhOsSNshQl0Qrd7qC9oQJzHvOL4qQXlQn2916+CXGywIjpGuIkoyZRRxHPiNQQ==", + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz", + "integrity": "sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==", "dev": true, "dependencies": { - "@babel/template": "^7.22.5", - "@babel/types": "^7.22.5" + "@babel/template": "^7.22.15", + "@babel/types": "^7.23.0" }, "engines": { "node": ">=6.9.0" @@ -401,9 +401,9 @@ } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.22.19", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.19.tgz", - "integrity": "sha512-Tinq7ybnEPFFXhlYOYFiSjespWQk0dq2dRNAiMdRTOYQzEGqnnNyrTxPYHP5r6wGjlF1rFgABdDV0g8EwD6Qbg==", + "version": "7.22.20", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz", + "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==", "dev": true, "engines": { "node": ">=6.9.0" @@ -494,9 +494,9 @@ } }, "node_modules/@babel/parser": { - "version": "7.22.16", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.22.16.tgz", - "integrity": "sha512-+gPfKv8UWeKKeJTUxe59+OobVcrYHETCsORl61EmSkmgymguYk/X5bp7GuUIXaFsc6y++v8ZxPsLSSuujqDphA==", + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.0.tgz", + "integrity": "sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw==", "dev": true, "bin": { "parser": "bin/babel-parser.js" @@ -638,19 +638,19 @@ } }, "node_modules/@babel/traverse": { - "version": "7.22.19", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.22.19.tgz", - "integrity": "sha512-ZCcpVPK64krfdScRbpxF6xA5fz7IOsfMwx1tcACvCzt6JY+0aHkBk7eIU8FRDSZRU5Zei6Z4JfgAxN1bqXGECg==", + "version": "7.23.2", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.2.tgz", + "integrity": "sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw==", "dev": true, "dependencies": { "@babel/code-frame": "^7.22.13", - "@babel/generator": "^7.22.15", - "@babel/helper-environment-visitor": "^7.22.5", - "@babel/helper-function-name": "^7.22.5", + "@babel/generator": "^7.23.0", + "@babel/helper-environment-visitor": "^7.22.20", + "@babel/helper-function-name": "^7.23.0", "@babel/helper-hoist-variables": "^7.22.5", "@babel/helper-split-export-declaration": "^7.22.6", - "@babel/parser": "^7.22.16", - "@babel/types": "^7.22.19", + "@babel/parser": "^7.23.0", + "@babel/types": "^7.23.0", "debug": "^4.1.0", "globals": "^11.1.0" }, @@ -668,13 +668,13 @@ } }, "node_modules/@babel/types": { - "version": "7.22.19", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.22.19.tgz", - "integrity": "sha512-P7LAw/LbojPzkgp5oznjE6tQEIWbp4PkkfrZDINTro9zgBRtI324/EYsiSI7lhPbpIQ+DCeR2NNmMWANGGfZsg==", + "version": "7.23.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.0.tgz", + "integrity": "sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg==", "dev": true, "dependencies": { "@babel/helper-string-parser": "^7.22.5", - "@babel/helper-validator-identifier": "^7.22.19", + "@babel/helper-validator-identifier": "^7.22.20", "to-fast-properties": "^2.0.0" }, "engines": { @@ -13812,9 +13812,9 @@ } }, "node_modules/postcss": { - "version": "8.4.29", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.29.tgz", - "integrity": "sha512-cbI+jaqIeu/VGqXEarWkRCCffhjgXc0qjBtXpqJhTBohMUjUQnbBr0xqX3vEKudc4iviTewcJo5ajcec5+wdJw==", + "version": "8.4.31", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", + "integrity": "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==", "dev": true, "funding": [ { diff --git a/package-lock.json b/package-lock.json index 40a870f..1995ec3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2183,9 +2183,9 @@ } }, "node_modules/crypto-js": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.1.1.tgz", - "integrity": "sha512-o2JlM7ydqd3Qk9CA0L4NL6mTzU2sdx96a+oOfPu8Mkl/PK51vSyoi8/rQ8NknZtk44vq15lmhAj9CIAGwgeWKw==" + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz", + "integrity": "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==" }, "node_modules/css-select": { "version": "5.1.0", @@ -5116,9 +5116,9 @@ } }, "node_modules/postcss": { - "version": "8.4.24", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.24.tgz", - "integrity": "sha512-M0RzbcI0sO/XJNucsGjvWU9ERWxb/ytp1w6dKtxTKgixdtQDq4rmx/g8W1hnaheq9jgwL/oyEdH5Bc4WwJKMqg==", + "version": "8.4.31", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", + "integrity": "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==", "dev": true, "funding": [ { From b868ec1f81a5cfa6f184f43e32b84363e013ff2f Mon Sep 17 00:00:00 2001 From: mrjvs Date: Thu, 26 Oct 2023 21:18:19 +0200 Subject: [PATCH 21/22] bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 04d1619..dbb0d40 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@movie-web/providers", - "version": "1.0.1", + "version": "1.0.2", "description": "Package that contains all the providers of movie-web", "main": "./lib/index.umd.js", "types": "./lib/index.d.ts", From 3e7d1f19d31bf2972c4e357b4d20dbef58e1a83e Mon Sep 17 00:00:00 2001 From: William Oldham Date: Thu, 26 Oct 2023 20:33:20 +0100 Subject: [PATCH 22/22] Update src/providers/sources/showbox/index.ts --- src/providers/sources/showbox/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/providers/sources/showbox/index.ts b/src/providers/sources/showbox/index.ts index 1534068..35d6fff 100644 --- a/src/providers/sources/showbox/index.ts +++ b/src/providers/sources/showbox/index.ts @@ -9,7 +9,7 @@ import { NotFoundError } from '@/utils/errors'; const showboxBase = `https://www.showbox.media`; export const showBoxScraper = makeSourcerer({ - id: 'showBox', + id: 'show_box', name: 'ShowBox', rank: 20, flags: [flags.NO_CORS],