diff --git a/package-lock.json b/package-lock.json index 498edb7..bf994c1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,8 @@ "crypto-js": "^4.1.1", "form-data": "^4.0.0", "node-fetch": "^2.7.0", - "randombytes": "^2.1.0" + "randombytes": "^2.1.0", + "unpacker": "^1.0.1" }, "devDependencies": { "@types/crypto-js": "^4.1.1", @@ -6162,6 +6163,11 @@ "node": ">= 4.0.0" } }, + "node_modules/unpacker": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/unpacker/-/unpacker-1.0.1.tgz", + "integrity": "sha512-0HTljwp8+JBdITpoHcK1LWi7X9U2BspUmWv78UWZh7NshYhbh1nec8baY/iSbe2OQTZ2bhAtVdnr6/BTD0DKVg==" + }, "node_modules/untildify": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz", diff --git a/package.json b/package.json index 5318b2e..2075025 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,7 @@ "eslint-import-resolver-typescript": "^3.5.5", "eslint-plugin-import": "^2.27.5", "eslint-plugin-prettier": "^4.2.1", + "node-fetch": "^2.7.0", "prettier": "^2.6.2", "spinnies": "^0.5.1", "ts-node": "^10.9.1", @@ -72,14 +73,14 @@ "vite": "^4.0.0", "vite-plugin-dts": "^3.5.3", "vite-plugin-eslint": "^1.8.1", - "vitest": "^0.32.2", - "node-fetch": "^2.7.0" + "vitest": "^0.32.2" }, "dependencies": { "cheerio": "^1.0.0-rc.12", "crypto-js": "^4.1.1", "form-data": "^4.0.0", "node-fetch": "^2.7.0", - "randombytes": "^2.1.0" + "randombytes": "^2.1.0", + "unpacker": "^1.0.1" } } diff --git a/src/providers/all.ts b/src/providers/all.ts index 968b61d..ef26094 100644 --- a/src/providers/all.ts +++ b/src/providers/all.ts @@ -1,19 +1,22 @@ import { Embed, Sourcerer } from '@/providers/base'; +import { mixdropScraper } from '@/providers/embeds/mixdrop'; import { mp4uploadScraper } from '@/providers/embeds/mp4upload'; import { streamsbScraper } from '@/providers/embeds/streamsb'; import { upcloudScraper } from '@/providers/embeds/upcloud'; +import { upstreamScraper } from '@/providers/embeds/upstream'; import { flixhqScraper } from '@/providers/sources/flixhq/index'; import { goMoviesScraper } from '@/providers/sources/gomovies/index'; import { kissAsianScraper } from '@/providers/sources/kissasian/index'; import { remotestreamScraper } from '@/providers/sources/remotestream'; import { superStreamScraper } from '@/providers/sources/superstream/index'; +import { zoechipScraper } from '@/providers/sources/zoechip'; export function gatherAllSources(): Array { // all sources are gathered here - return [flixhqScraper, remotestreamScraper, kissAsianScraper, superStreamScraper, goMoviesScraper]; + return [flixhqScraper, remotestreamScraper, kissAsianScraper, superStreamScraper, goMoviesScraper, zoechipScraper]; } export function gatherAllEmbeds(): Array { // all embeds are gathered here - return [upcloudScraper, mp4uploadScraper, streamsbScraper]; + return [upcloudScraper, mp4uploadScraper, streamsbScraper, upstreamScraper, mixdropScraper]; } diff --git a/src/providers/embeds/mixdrop.ts b/src/providers/embeds/mixdrop.ts new file mode 100644 index 0000000..e7df2cf --- /dev/null +++ b/src/providers/embeds/mixdrop.ts @@ -0,0 +1,52 @@ +import * as unpacker from 'unpacker'; + +import { makeEmbed } from '@/providers/base'; + +const packedRegex = /(eval\(function\(p,a,c,k,e,d\){.*{}\)\))/; +const linkRegex = /MDCore\.wurl="(.*?)";/; + +export const mixdropScraper = makeEmbed({ + id: 'mixdrop', + name: 'MixDrop', + rank: 198, + async scrape(ctx) { + // Example url: https://mixdrop.co/e/pkwrgp0pizgod0 + // Example url: https://mixdrop.vc/e/pkwrgp0pizgod0 + const streamRes = await ctx.proxiedFetcher(ctx.url); + const packed = streamRes.match(packedRegex); + + // MixDrop uses a queue system for embeds + // If an embed is too new, the queue will + // not be completed and thus the packed + // JavaScript not present + if (!packed) { + throw new Error('failed to find packed mixdrop JavaScript'); + } + + const unpacked = unpacker.unpack(packed[1]); + const link = unpacked.match(linkRegex); + + if (!link) { + throw new Error('failed to find packed mixdrop source link'); + } + + const url = link[1]; + + return { + stream: { + type: 'file', + flags: [], + qualities: { + unknown: { + type: 'mp4', + url: url.startsWith('http') ? url : `https:${url}`, // URLs don't always start with the protocol + headers: { + // MixDrop requires this header on all streams + Referer: 'https://mixdrop.co/', + }, + }, + }, + }, + }; + }, +}); diff --git a/src/providers/embeds/upstream.ts b/src/providers/embeds/upstream.ts new file mode 100644 index 0000000..62d2f01 --- /dev/null +++ b/src/providers/embeds/upstream.ts @@ -0,0 +1,35 @@ +import * as unpacker from 'unpacker'; + +import { flags } from '@/main/targets'; +import { makeEmbed } from '@/providers/base'; + +const packedRegex = /(eval\(function\(p,a,c,k,e,d\).*\)\)\))/; +const linkRegex = /sources:\[{file:"(.*?)"/; + +export const upstreamScraper = makeEmbed({ + id: 'upstream', + name: 'UpStream', + rank: 199, + async scrape(ctx) { + // Example url: https://upstream.to/embed-omscqgn6jc8r.html + const streamRes = await ctx.proxiedFetcher(ctx.url); + const packed = streamRes.match(packedRegex); + + if (packed) { + const unpacked = unpacker.unpack(packed[1]); + const link = unpacked.match(linkRegex); + + if (link) { + return { + stream: { + type: 'hls', + playlist: link[1], + flags: [flags.NO_CORS], + }, + }; + } + } + + throw new Error('upstream source not found'); + }, +}); diff --git a/src/providers/sources/zoechip/common.ts b/src/providers/sources/zoechip/common.ts new file mode 100644 index 0000000..7ca1b2e --- /dev/null +++ b/src/providers/sources/zoechip/common.ts @@ -0,0 +1,71 @@ +import { MovieMedia, ShowMedia } from '@/main/media'; +import { mixdropScraper } from '@/providers/embeds/mixdrop'; +import { upcloudScraper } from '@/providers/embeds/upcloud'; +import { upstreamScraper } from '@/providers/embeds/upstream'; +import { getZoeChipSourceURL, getZoeChipSources } from '@/providers/sources/zoechip/scrape'; +import { ScrapeContext } from '@/utils/context'; + +export const zoeBase = 'https://zoechip.cc'; + +export type MovieContext = ScrapeContext & { + media: MovieMedia; +}; + +export type ShowContext = ScrapeContext & { + media: ShowMedia; +}; + +export type ZoeChipSourceDetails = { + type: string; // Only seen "iframe" so far + link: string; + sources: string[]; // Never seen this populated, assuming it's a string array + tracks: string[]; // Never seen this populated, assuming it's a string array + title: string; +}; + +export async function formatSource(ctx: MovieContext | ShowContext, source: { embed: string; episodeId: string }) { + const link = await getZoeChipSourceURL(ctx, source.episodeId); + if (link) { + const embed = { + embedId: '', + url: link, + }; + + const parsedUrl = new URL(link); + + switch (parsedUrl.host) { + case 'rabbitstream.net': + embed.embedId = upcloudScraper.id; + break; + case 'upstream.to': + embed.embedId = upstreamScraper.id; + break; + case 'mixdrop.co': + embed.embedId = mixdropScraper.id; + break; + default: + throw new Error(`Failed to find ZoeChip embed source for ${link}`); + } + + return embed; + } +} + +export async function createZoeChipStreamData(ctx: MovieContext | ShowContext, id: string) { + const sources = await getZoeChipSources(ctx, id); + const embeds: { + embedId: string; + url: string; + }[] = []; + + for (const source of sources) { + const formatted = await formatSource(ctx, source); + if (formatted) { + embeds.push(formatted); + } + } + + return { + embeds, + }; +} diff --git a/src/providers/sources/zoechip/index.ts b/src/providers/sources/zoechip/index.ts new file mode 100644 index 0000000..94593c7 --- /dev/null +++ b/src/providers/sources/zoechip/index.ts @@ -0,0 +1,13 @@ +import { flags } from '@/main/targets'; +import { makeSourcerer } from '@/providers/base'; +import { scrapeMovie } from '@/providers/sources/zoechip/scrape-movie'; +import { scrapeShow } from '@/providers/sources/zoechip/scrape-show'; + +export const zoechipScraper = makeSourcerer({ + id: 'zoechip', + name: 'ZoeChip', + rank: 110, + flags: [flags.NO_CORS], + scrapeMovie, + scrapeShow, +}); diff --git a/src/providers/sources/zoechip/scrape-movie.ts b/src/providers/sources/zoechip/scrape-movie.ts new file mode 100644 index 0000000..448af0b --- /dev/null +++ b/src/providers/sources/zoechip/scrape-movie.ts @@ -0,0 +1,12 @@ +import { MovieContext, createZoeChipStreamData } from '@/providers/sources/zoechip/common'; +import { getZoeChipMovieID } from '@/providers/sources/zoechip/search'; +import { NotFoundError } from '@/utils/errors'; + +export async function scrapeMovie(ctx: MovieContext) { + const movieID = await getZoeChipMovieID(ctx, ctx.media); + if (!movieID) { + throw new NotFoundError('no search results match'); + } + + return createZoeChipStreamData(ctx, movieID); +} diff --git a/src/providers/sources/zoechip/scrape-show.ts b/src/providers/sources/zoechip/scrape-show.ts new file mode 100644 index 0000000..eb21d13 --- /dev/null +++ b/src/providers/sources/zoechip/scrape-show.ts @@ -0,0 +1,23 @@ +import { ShowContext, createZoeChipStreamData } from '@/providers/sources/zoechip/common'; +import { getZoeChipEpisodeID, getZoeChipSeasonID } from '@/providers/sources/zoechip/scrape'; +import { getZoeChipShowID } from '@/providers/sources/zoechip/search'; +import { NotFoundError } from '@/utils/errors'; + +export async function scrapeShow(ctx: ShowContext) { + const showID = await getZoeChipShowID(ctx, ctx.media); + if (!showID) { + throw new NotFoundError('no search results match'); + } + + const seasonID = await getZoeChipSeasonID(ctx, ctx.media, showID); + if (!seasonID) { + throw new NotFoundError('no season found'); + } + + const episodeID = await getZoeChipEpisodeID(ctx, ctx.media, seasonID); + if (!episodeID) { + throw new NotFoundError('no episode found'); + } + + return createZoeChipStreamData(ctx, episodeID); +} diff --git a/src/providers/sources/zoechip/scrape.ts b/src/providers/sources/zoechip/scrape.ts new file mode 100644 index 0000000..5be0edd --- /dev/null +++ b/src/providers/sources/zoechip/scrape.ts @@ -0,0 +1,126 @@ +import { load } from 'cheerio'; + +import { ShowMedia } from '@/main/media'; +import { MovieContext, ShowContext, ZoeChipSourceDetails, zoeBase } from '@/providers/sources/zoechip/common'; +import { ScrapeContext } from '@/utils/context'; + +export async function getZoeChipSources(ctx: MovieContext | ShowContext, id: string) { + // Movies use /ajax/episode/list/ID + // Shows use /ajax/episode/servers/ID + const endpoint = ctx.media.type === 'movie' ? 'list' : 'servers'; + const html = await ctx.proxiedFetcher(`/ajax/episode/${endpoint}/${id}`, { + baseUrl: zoeBase, + }); + const $ = load(html); + + return $('.nav-item a') + .toArray() + .map((el) => { + // Movies use data-linkid + // Shows use data-id + const idAttribute = ctx.media.type === 'movie' ? 'data-linkid' : 'data-id'; + const element = $(el); + const embedTitle = element.attr('title'); + const linkId = element.attr(idAttribute); + + if (!embedTitle || !linkId) { + throw new Error('invalid sources'); + } + + return { + embed: embedTitle, + episodeId: linkId, + }; + }); +} + +export async function getZoeChipSourceURL(ctx: ScrapeContext, sourceID: string): Promise { + const details = await ctx.proxiedFetcher(`/ajax/sources/${sourceID}`, { + baseUrl: zoeBase, + }); + + // TODO - Support non-iframe sources + if (details.type !== 'iframe') { + return null; + } + + // TODO - Extract the other data from the source + + return details.link; +} + +export async function getZoeChipSeasonID(ctx: ScrapeContext, media: ShowMedia, showID: string): Promise { + const html = await ctx.proxiedFetcher(`/ajax/season/list/${showID}`, { + baseUrl: zoeBase, + }); + + const $ = load(html); + + const seasons = $('.dropdown-menu a') + .toArray() + .map((el) => { + const element = $(el); + const seasonID = element.attr('data-id'); + const seasonNumber = element.html()?.split(' ')[1]; + + if (!seasonID || !seasonNumber || Number.isNaN(Number(seasonNumber))) { + throw new Error('invalid season'); + } + + return { + id: seasonID, + season: Number(seasonNumber), + }; + }); + + const foundSeason = seasons.find((season) => season.season === media.season.number); + + if (!foundSeason) { + return null; + } + + return foundSeason.id; +} + +export async function getZoeChipEpisodeID( + ctx: ScrapeContext, + media: ShowMedia, + seasonID: string, +): Promise { + const episodeNumberRegex = /Eps (\d*):/; + const html = await ctx.proxiedFetcher(`/ajax/season/episodes/${seasonID}`, { + baseUrl: zoeBase, + }); + + const $ = load(html); + + const episodes = $('.eps-item') + .toArray() + .map((el) => { + const element = $(el); + const episodeID = element.attr('data-id'); + const title = element.attr('title'); + + if (!episodeID || !title) { + throw new Error('invalid episode'); + } + + const regexResult = title.match(episodeNumberRegex); + if (!regexResult || Number.isNaN(Number(regexResult[1]))) { + throw new Error('invalid episode'); + } + + return { + id: episodeID, + episode: Number(regexResult[1]), + }; + }); + + const foundEpisode = episodes.find((episode) => episode.episode === media.episode.number); + + if (!foundEpisode) { + return null; + } + + return foundEpisode.id; +} diff --git a/src/providers/sources/zoechip/search.ts b/src/providers/sources/zoechip/search.ts new file mode 100644 index 0000000..6297bc4 --- /dev/null +++ b/src/providers/sources/zoechip/search.ts @@ -0,0 +1,111 @@ +import { load } from 'cheerio'; + +import { MovieMedia, ShowMedia } from '@/main/media'; +import { zoeBase } from '@/providers/sources/zoechip/common'; +import { compareMedia } from '@/utils/compare'; +import { ScrapeContext } from '@/utils/context'; + +export async function getZoeChipSearchResults(ctx: ScrapeContext, media: MovieMedia | ShowMedia) { + const titleCleaned = media.title.toLocaleLowerCase().replace(/ /g, '-'); + + const html = await ctx.proxiedFetcher(`/search/${titleCleaned}`, { + baseUrl: zoeBase, + }); + + const $ = load(html); + return $('.film_list-wrap .flw-item .film-detail') + .toArray() + .map((element) => { + const movie = $(element); + const anchor = movie.find('.film-name a'); + const info = movie.find('.fd-infor'); + + const title = anchor.attr('title'); + const href = anchor.attr('href'); + const type = info.find('.fdi-type').html(); + let year = info.find('.fdi-item').html(); + const id = href?.split('-').pop(); + + if (!title) { + return null; + } + + if (!href) { + return null; + } + + if (!type) { + return null; + } + + // TV shows on ZoeChip do not have a year in their search results + // Allow TV shows to pass this failure + if (!year || Number.isNaN(Number(year))) { + if (type === 'TV') { + year = '0'; + } else { + return null; + } + } + + if (!id) { + return null; + } + + return { + title, + year: Number(year), + id, + type, + href, + }; + }); +} + +export async function getZoeChipMovieID(ctx: ScrapeContext, media: MovieMedia): Promise { + const searchResults = await getZoeChipSearchResults(ctx, media); + + const matchingItem = searchResults.find((v) => v && v.type === 'Movie' && compareMedia(media, v.title, v.year)); + + if (!matchingItem) { + return null; + } + + return matchingItem.id; +} + +export async function getZoeChipShowID(ctx: ScrapeContext, media: ShowMedia): Promise { + // ZoeChip TV shows don't have a year on their search results + // This makes it hard to filter between shows with the same name + // To find the year, we must query each shows details page + // This is slower, but more reliable + + const releasedRegex = /<\/strong><\/span> (\d.*)-\d.*-\d.*/; + const searchResults = await getZoeChipSearchResults(ctx, media); + + const filtered = searchResults.filter((v) => v && v.type === 'TV' && compareMedia(media, v.title)); + + for (const result of filtered) { + // This gets filtered above but the linter Gods don't think so + if (!result) { + continue; + } + + const html = await ctx.proxiedFetcher(result.href, { + baseUrl: zoeBase, + }); + + // The HTML is not structured in a way that makes using Cheerio clean + // There are no unique IDs or classes to query, resulting in long ugly queries + // Regex is faster and cleaner in this case + const regexResult = html.match(releasedRegex); + if (regexResult) { + const year = Number(regexResult[1]); + if (!Number.isNaN(year) && compareMedia(media, result.title, year)) { + return result.id; + } + } + } + + return null; +} diff --git a/src/providers/streams.ts b/src/providers/streams.ts index 54b3cb1..8a4674c 100644 --- a/src/providers/streams.ts +++ b/src/providers/streams.ts @@ -3,9 +3,10 @@ import { Flags } from '@/main/targets'; export type StreamFile = { type: 'mp4'; url: string; + headers?: Record; }; -export type Qualities = '360' | '480' | '720' | '1080'; +export type Qualities = 'unknown' | '360' | '480' | '720' | '1080'; export type FileBasedStream = { type: 'file';