Merge pull request #12 from jonbarrow/dev
Add ZoeChip source & mixdrop and upstream embeds
This commit is contained in:
commit
123c5cc142
|
@ -13,7 +13,8 @@
|
|||
"crypto-js": "^4.1.1",
|
||||
"form-data": "^4.0.0",
|
||||
"node-fetch": "^2.7.0",
|
||||
"randombytes": "^2.1.0"
|
||||
"randombytes": "^2.1.0",
|
||||
"unpacker": "^1.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/crypto-js": "^4.1.1",
|
||||
|
@ -6162,6 +6163,11 @@
|
|||
"node": ">= 4.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/unpacker": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/unpacker/-/unpacker-1.0.1.tgz",
|
||||
"integrity": "sha512-0HTljwp8+JBdITpoHcK1LWi7X9U2BspUmWv78UWZh7NshYhbh1nec8baY/iSbe2OQTZ2bhAtVdnr6/BTD0DKVg=="
|
||||
},
|
||||
"node_modules/untildify": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz",
|
||||
|
|
|
@ -63,6 +63,7 @@
|
|||
"eslint-import-resolver-typescript": "^3.5.5",
|
||||
"eslint-plugin-import": "^2.27.5",
|
||||
"eslint-plugin-prettier": "^4.2.1",
|
||||
"node-fetch": "^2.7.0",
|
||||
"prettier": "^2.6.2",
|
||||
"spinnies": "^0.5.1",
|
||||
"ts-node": "^10.9.1",
|
||||
|
@ -72,14 +73,14 @@
|
|||
"vite": "^4.0.0",
|
||||
"vite-plugin-dts": "^3.5.3",
|
||||
"vite-plugin-eslint": "^1.8.1",
|
||||
"vitest": "^0.32.2",
|
||||
"node-fetch": "^2.7.0"
|
||||
"vitest": "^0.32.2"
|
||||
},
|
||||
"dependencies": {
|
||||
"cheerio": "^1.0.0-rc.12",
|
||||
"crypto-js": "^4.1.1",
|
||||
"form-data": "^4.0.0",
|
||||
"node-fetch": "^2.7.0",
|
||||
"randombytes": "^2.1.0"
|
||||
"randombytes": "^2.1.0",
|
||||
"unpacker": "^1.0.1"
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,19 +1,22 @@
|
|||
import { Embed, Sourcerer } from '@/providers/base';
|
||||
import { mixdropScraper } from '@/providers/embeds/mixdrop';
|
||||
import { mp4uploadScraper } from '@/providers/embeds/mp4upload';
|
||||
import { streamsbScraper } from '@/providers/embeds/streamsb';
|
||||
import { upcloudScraper } from '@/providers/embeds/upcloud';
|
||||
import { upstreamScraper } from '@/providers/embeds/upstream';
|
||||
import { flixhqScraper } from '@/providers/sources/flixhq/index';
|
||||
import { goMoviesScraper } from '@/providers/sources/gomovies/index';
|
||||
import { kissAsianScraper } from '@/providers/sources/kissasian/index';
|
||||
import { remotestreamScraper } from '@/providers/sources/remotestream';
|
||||
import { superStreamScraper } from '@/providers/sources/superstream/index';
|
||||
import { zoechipScraper } from '@/providers/sources/zoechip';
|
||||
|
||||
export function gatherAllSources(): Array<Sourcerer> {
|
||||
// all sources are gathered here
|
||||
return [flixhqScraper, remotestreamScraper, kissAsianScraper, superStreamScraper, goMoviesScraper];
|
||||
return [flixhqScraper, remotestreamScraper, kissAsianScraper, superStreamScraper, goMoviesScraper, zoechipScraper];
|
||||
}
|
||||
|
||||
export function gatherAllEmbeds(): Array<Embed> {
|
||||
// all embeds are gathered here
|
||||
return [upcloudScraper, mp4uploadScraper, streamsbScraper];
|
||||
return [upcloudScraper, mp4uploadScraper, streamsbScraper, upstreamScraper, mixdropScraper];
|
||||
}
|
||||
|
|
|
@ -0,0 +1,52 @@
|
|||
import * as unpacker from 'unpacker';
|
||||
|
||||
import { makeEmbed } from '@/providers/base';
|
||||
|
||||
const packedRegex = /(eval\(function\(p,a,c,k,e,d\){.*{}\)\))/;
|
||||
const linkRegex = /MDCore\.wurl="(.*?)";/;
|
||||
|
||||
export const mixdropScraper = makeEmbed({
|
||||
id: 'mixdrop',
|
||||
name: 'MixDrop',
|
||||
rank: 198,
|
||||
async scrape(ctx) {
|
||||
// Example url: https://mixdrop.co/e/pkwrgp0pizgod0
|
||||
// Example url: https://mixdrop.vc/e/pkwrgp0pizgod0
|
||||
const streamRes = await ctx.proxiedFetcher<string>(ctx.url);
|
||||
const packed = streamRes.match(packedRegex);
|
||||
|
||||
// MixDrop uses a queue system for embeds
|
||||
// If an embed is too new, the queue will
|
||||
// not be completed and thus the packed
|
||||
// JavaScript not present
|
||||
if (!packed) {
|
||||
throw new Error('failed to find packed mixdrop JavaScript');
|
||||
}
|
||||
|
||||
const unpacked = unpacker.unpack(packed[1]);
|
||||
const link = unpacked.match(linkRegex);
|
||||
|
||||
if (!link) {
|
||||
throw new Error('failed to find packed mixdrop source link');
|
||||
}
|
||||
|
||||
const url = link[1];
|
||||
|
||||
return {
|
||||
stream: {
|
||||
type: 'file',
|
||||
flags: [],
|
||||
qualities: {
|
||||
unknown: {
|
||||
type: 'mp4',
|
||||
url: url.startsWith('http') ? url : `https:${url}`, // URLs don't always start with the protocol
|
||||
headers: {
|
||||
// MixDrop requires this header on all streams
|
||||
Referer: 'https://mixdrop.co/',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
|
@ -0,0 +1,35 @@
|
|||
import * as unpacker from 'unpacker';
|
||||
|
||||
import { flags } from '@/main/targets';
|
||||
import { makeEmbed } from '@/providers/base';
|
||||
|
||||
const packedRegex = /(eval\(function\(p,a,c,k,e,d\).*\)\)\))/;
|
||||
const linkRegex = /sources:\[{file:"(.*?)"/;
|
||||
|
||||
export const upstreamScraper = makeEmbed({
|
||||
id: 'upstream',
|
||||
name: 'UpStream',
|
||||
rank: 199,
|
||||
async scrape(ctx) {
|
||||
// Example url: https://upstream.to/embed-omscqgn6jc8r.html
|
||||
const streamRes = await ctx.proxiedFetcher<string>(ctx.url);
|
||||
const packed = streamRes.match(packedRegex);
|
||||
|
||||
if (packed) {
|
||||
const unpacked = unpacker.unpack(packed[1]);
|
||||
const link = unpacked.match(linkRegex);
|
||||
|
||||
if (link) {
|
||||
return {
|
||||
stream: {
|
||||
type: 'hls',
|
||||
playlist: link[1],
|
||||
flags: [flags.NO_CORS],
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('upstream source not found');
|
||||
},
|
||||
});
|
|
@ -0,0 +1,71 @@
|
|||
import { MovieMedia, ShowMedia } from '@/main/media';
|
||||
import { mixdropScraper } from '@/providers/embeds/mixdrop';
|
||||
import { upcloudScraper } from '@/providers/embeds/upcloud';
|
||||
import { upstreamScraper } from '@/providers/embeds/upstream';
|
||||
import { getZoeChipSourceURL, getZoeChipSources } from '@/providers/sources/zoechip/scrape';
|
||||
import { ScrapeContext } from '@/utils/context';
|
||||
|
||||
export const zoeBase = 'https://zoechip.cc';
|
||||
|
||||
export type MovieContext = ScrapeContext & {
|
||||
media: MovieMedia;
|
||||
};
|
||||
|
||||
export type ShowContext = ScrapeContext & {
|
||||
media: ShowMedia;
|
||||
};
|
||||
|
||||
export type ZoeChipSourceDetails = {
|
||||
type: string; // Only seen "iframe" so far
|
||||
link: string;
|
||||
sources: string[]; // Never seen this populated, assuming it's a string array
|
||||
tracks: string[]; // Never seen this populated, assuming it's a string array
|
||||
title: string;
|
||||
};
|
||||
|
||||
export async function formatSource(ctx: MovieContext | ShowContext, source: { embed: string; episodeId: string }) {
|
||||
const link = await getZoeChipSourceURL(ctx, source.episodeId);
|
||||
if (link) {
|
||||
const embed = {
|
||||
embedId: '',
|
||||
url: link,
|
||||
};
|
||||
|
||||
const parsedUrl = new URL(link);
|
||||
|
||||
switch (parsedUrl.host) {
|
||||
case 'rabbitstream.net':
|
||||
embed.embedId = upcloudScraper.id;
|
||||
break;
|
||||
case 'upstream.to':
|
||||
embed.embedId = upstreamScraper.id;
|
||||
break;
|
||||
case 'mixdrop.co':
|
||||
embed.embedId = mixdropScraper.id;
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Failed to find ZoeChip embed source for ${link}`);
|
||||
}
|
||||
|
||||
return embed;
|
||||
}
|
||||
}
|
||||
|
||||
export async function createZoeChipStreamData(ctx: MovieContext | ShowContext, id: string) {
|
||||
const sources = await getZoeChipSources(ctx, id);
|
||||
const embeds: {
|
||||
embedId: string;
|
||||
url: string;
|
||||
}[] = [];
|
||||
|
||||
for (const source of sources) {
|
||||
const formatted = await formatSource(ctx, source);
|
||||
if (formatted) {
|
||||
embeds.push(formatted);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
embeds,
|
||||
};
|
||||
}
|
|
@ -0,0 +1,13 @@
|
|||
import { flags } from '@/main/targets';
|
||||
import { makeSourcerer } from '@/providers/base';
|
||||
import { scrapeMovie } from '@/providers/sources/zoechip/scrape-movie';
|
||||
import { scrapeShow } from '@/providers/sources/zoechip/scrape-show';
|
||||
|
||||
export const zoechipScraper = makeSourcerer({
|
||||
id: 'zoechip',
|
||||
name: 'ZoeChip',
|
||||
rank: 110,
|
||||
flags: [flags.NO_CORS],
|
||||
scrapeMovie,
|
||||
scrapeShow,
|
||||
});
|
|
@ -0,0 +1,12 @@
|
|||
import { MovieContext, createZoeChipStreamData } from '@/providers/sources/zoechip/common';
|
||||
import { getZoeChipMovieID } from '@/providers/sources/zoechip/search';
|
||||
import { NotFoundError } from '@/utils/errors';
|
||||
|
||||
export async function scrapeMovie(ctx: MovieContext) {
|
||||
const movieID = await getZoeChipMovieID(ctx, ctx.media);
|
||||
if (!movieID) {
|
||||
throw new NotFoundError('no search results match');
|
||||
}
|
||||
|
||||
return createZoeChipStreamData(ctx, movieID);
|
||||
}
|
|
@ -0,0 +1,23 @@
|
|||
import { ShowContext, createZoeChipStreamData } from '@/providers/sources/zoechip/common';
|
||||
import { getZoeChipEpisodeID, getZoeChipSeasonID } from '@/providers/sources/zoechip/scrape';
|
||||
import { getZoeChipShowID } from '@/providers/sources/zoechip/search';
|
||||
import { NotFoundError } from '@/utils/errors';
|
||||
|
||||
export async function scrapeShow(ctx: ShowContext) {
|
||||
const showID = await getZoeChipShowID(ctx, ctx.media);
|
||||
if (!showID) {
|
||||
throw new NotFoundError('no search results match');
|
||||
}
|
||||
|
||||
const seasonID = await getZoeChipSeasonID(ctx, ctx.media, showID);
|
||||
if (!seasonID) {
|
||||
throw new NotFoundError('no season found');
|
||||
}
|
||||
|
||||
const episodeID = await getZoeChipEpisodeID(ctx, ctx.media, seasonID);
|
||||
if (!episodeID) {
|
||||
throw new NotFoundError('no episode found');
|
||||
}
|
||||
|
||||
return createZoeChipStreamData(ctx, episodeID);
|
||||
}
|
|
@ -0,0 +1,126 @@
|
|||
import { load } from 'cheerio';
|
||||
|
||||
import { ShowMedia } from '@/main/media';
|
||||
import { MovieContext, ShowContext, ZoeChipSourceDetails, zoeBase } from '@/providers/sources/zoechip/common';
|
||||
import { ScrapeContext } from '@/utils/context';
|
||||
|
||||
export async function getZoeChipSources(ctx: MovieContext | ShowContext, id: string) {
|
||||
// Movies use /ajax/episode/list/ID
|
||||
// Shows use /ajax/episode/servers/ID
|
||||
const endpoint = ctx.media.type === 'movie' ? 'list' : 'servers';
|
||||
const html = await ctx.proxiedFetcher<string>(`/ajax/episode/${endpoint}/${id}`, {
|
||||
baseUrl: zoeBase,
|
||||
});
|
||||
const $ = load(html);
|
||||
|
||||
return $('.nav-item a')
|
||||
.toArray()
|
||||
.map((el) => {
|
||||
// Movies use data-linkid
|
||||
// Shows use data-id
|
||||
const idAttribute = ctx.media.type === 'movie' ? 'data-linkid' : 'data-id';
|
||||
const element = $(el);
|
||||
const embedTitle = element.attr('title');
|
||||
const linkId = element.attr(idAttribute);
|
||||
|
||||
if (!embedTitle || !linkId) {
|
||||
throw new Error('invalid sources');
|
||||
}
|
||||
|
||||
return {
|
||||
embed: embedTitle,
|
||||
episodeId: linkId,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export async function getZoeChipSourceURL(ctx: ScrapeContext, sourceID: string): Promise<string | null> {
|
||||
const details = await ctx.proxiedFetcher<ZoeChipSourceDetails>(`/ajax/sources/${sourceID}`, {
|
||||
baseUrl: zoeBase,
|
||||
});
|
||||
|
||||
// TODO - Support non-iframe sources
|
||||
if (details.type !== 'iframe') {
|
||||
return null;
|
||||
}
|
||||
|
||||
// TODO - Extract the other data from the source
|
||||
|
||||
return details.link;
|
||||
}
|
||||
|
||||
export async function getZoeChipSeasonID(ctx: ScrapeContext, media: ShowMedia, showID: string): Promise<string | null> {
|
||||
const html = await ctx.proxiedFetcher<string>(`/ajax/season/list/${showID}`, {
|
||||
baseUrl: zoeBase,
|
||||
});
|
||||
|
||||
const $ = load(html);
|
||||
|
||||
const seasons = $('.dropdown-menu a')
|
||||
.toArray()
|
||||
.map((el) => {
|
||||
const element = $(el);
|
||||
const seasonID = element.attr('data-id');
|
||||
const seasonNumber = element.html()?.split(' ')[1];
|
||||
|
||||
if (!seasonID || !seasonNumber || Number.isNaN(Number(seasonNumber))) {
|
||||
throw new Error('invalid season');
|
||||
}
|
||||
|
||||
return {
|
||||
id: seasonID,
|
||||
season: Number(seasonNumber),
|
||||
};
|
||||
});
|
||||
|
||||
const foundSeason = seasons.find((season) => season.season === media.season.number);
|
||||
|
||||
if (!foundSeason) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return foundSeason.id;
|
||||
}
|
||||
|
||||
export async function getZoeChipEpisodeID(
|
||||
ctx: ScrapeContext,
|
||||
media: ShowMedia,
|
||||
seasonID: string,
|
||||
): Promise<string | null> {
|
||||
const episodeNumberRegex = /Eps (\d*):/;
|
||||
const html = await ctx.proxiedFetcher<string>(`/ajax/season/episodes/${seasonID}`, {
|
||||
baseUrl: zoeBase,
|
||||
});
|
||||
|
||||
const $ = load(html);
|
||||
|
||||
const episodes = $('.eps-item')
|
||||
.toArray()
|
||||
.map((el) => {
|
||||
const element = $(el);
|
||||
const episodeID = element.attr('data-id');
|
||||
const title = element.attr('title');
|
||||
|
||||
if (!episodeID || !title) {
|
||||
throw new Error('invalid episode');
|
||||
}
|
||||
|
||||
const regexResult = title.match(episodeNumberRegex);
|
||||
if (!regexResult || Number.isNaN(Number(regexResult[1]))) {
|
||||
throw new Error('invalid episode');
|
||||
}
|
||||
|
||||
return {
|
||||
id: episodeID,
|
||||
episode: Number(regexResult[1]),
|
||||
};
|
||||
});
|
||||
|
||||
const foundEpisode = episodes.find((episode) => episode.episode === media.episode.number);
|
||||
|
||||
if (!foundEpisode) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return foundEpisode.id;
|
||||
}
|
|
@ -0,0 +1,111 @@
|
|||
import { load } from 'cheerio';
|
||||
|
||||
import { MovieMedia, ShowMedia } from '@/main/media';
|
||||
import { zoeBase } from '@/providers/sources/zoechip/common';
|
||||
import { compareMedia } from '@/utils/compare';
|
||||
import { ScrapeContext } from '@/utils/context';
|
||||
|
||||
export async function getZoeChipSearchResults(ctx: ScrapeContext, media: MovieMedia | ShowMedia) {
|
||||
const titleCleaned = media.title.toLocaleLowerCase().replace(/ /g, '-');
|
||||
|
||||
const html = await ctx.proxiedFetcher<string>(`/search/${titleCleaned}`, {
|
||||
baseUrl: zoeBase,
|
||||
});
|
||||
|
||||
const $ = load(html);
|
||||
return $('.film_list-wrap .flw-item .film-detail')
|
||||
.toArray()
|
||||
.map((element) => {
|
||||
const movie = $(element);
|
||||
const anchor = movie.find('.film-name a');
|
||||
const info = movie.find('.fd-infor');
|
||||
|
||||
const title = anchor.attr('title');
|
||||
const href = anchor.attr('href');
|
||||
const type = info.find('.fdi-type').html();
|
||||
let year = info.find('.fdi-item').html();
|
||||
const id = href?.split('-').pop();
|
||||
|
||||
if (!title) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!href) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!type) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// TV shows on ZoeChip do not have a year in their search results
|
||||
// Allow TV shows to pass this failure
|
||||
if (!year || Number.isNaN(Number(year))) {
|
||||
if (type === 'TV') {
|
||||
year = '0';
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
if (!id) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
title,
|
||||
year: Number(year),
|
||||
id,
|
||||
type,
|
||||
href,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export async function getZoeChipMovieID(ctx: ScrapeContext, media: MovieMedia): Promise<string | null> {
|
||||
const searchResults = await getZoeChipSearchResults(ctx, media);
|
||||
|
||||
const matchingItem = searchResults.find((v) => v && v.type === 'Movie' && compareMedia(media, v.title, v.year));
|
||||
|
||||
if (!matchingItem) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return matchingItem.id;
|
||||
}
|
||||
|
||||
export async function getZoeChipShowID(ctx: ScrapeContext, media: ShowMedia): Promise<string | null> {
|
||||
// ZoeChip TV shows don't have a year on their search results
|
||||
// This makes it hard to filter between shows with the same name
|
||||
// To find the year, we must query each shows details page
|
||||
// This is slower, but more reliable
|
||||
|
||||
const releasedRegex = /<\/strong><\/span> (\d.*)-\d.*-\d.*/;
|
||||
const searchResults = await getZoeChipSearchResults(ctx, media);
|
||||
|
||||
const filtered = searchResults.filter((v) => v && v.type === 'TV' && compareMedia(media, v.title));
|
||||
|
||||
for (const result of filtered) {
|
||||
// This gets filtered above but the linter Gods don't think so
|
||||
if (!result) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const html = await ctx.proxiedFetcher<string>(result.href, {
|
||||
baseUrl: zoeBase,
|
||||
});
|
||||
|
||||
// The HTML is not structured in a way that makes using Cheerio clean
|
||||
// There are no unique IDs or classes to query, resulting in long ugly queries
|
||||
// Regex is faster and cleaner in this case
|
||||
const regexResult = html.match(releasedRegex);
|
||||
if (regexResult) {
|
||||
const year = Number(regexResult[1]);
|
||||
if (!Number.isNaN(year) && compareMedia(media, result.title, year)) {
|
||||
return result.id;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
|
@ -3,9 +3,10 @@ import { Flags } from '@/main/targets';
|
|||
export type StreamFile = {
|
||||
type: 'mp4';
|
||||
url: string;
|
||||
headers?: Record<string, string>;
|
||||
};
|
||||
|
||||
export type Qualities = '360' | '480' | '720' | '1080';
|
||||
export type Qualities = 'unknown' | '360' | '480' | '720' | '1080';
|
||||
|
||||
export type FileBasedStream = {
|
||||
type: 'file';
|
||||
|
|
Loading…
Reference in New Issue