dev
/
providers
mirror of https://github.com/sussy-code/providers.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/dev'

This commit is contained in:
Captain Jack Sparrow 2024-05-03 02:08:44 +00:00
parent fe04267494 ff3ceecc37
commit 749767ef4d
13 changed files with 381 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/dev-cli/browser/.gitignore vendored
View File

@ -1 +0,0 @@
dist

11
src/dev-cli/browser/index.html
View File

@ -1,11 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Scraper CLI</title>
</head>
<body>
<script src="./index.ts" type="module"></script>
</body>
</html>

17
src/dev-cli/browser/index.ts
View File

@ -1,17 +0,0 @@
import { makeProviders, makeSimpleProxyFetcher, makeStandardFetcher, targets } from '../../../lib';
(window as any).scrape = (proxyUrl: string, type: 'source' | 'embed', input: any) => {
const providers = makeProviders({
fetcher: makeStandardFetcher(fetch),
target: targets.BROWSER,
proxiedFetcher: makeSimpleProxyFetcher(proxyUrl, fetch),
});
if (type === 'source') {
return providers.runSourceScraper(input);
}
if (type === 'embed') {
return providers.runEmbedScraper(input);
}
throw new Error('Input input type');
};

12
src/providers/all.ts
View File

@ -17,13 +17,18 @@ import { goMoviesScraper } from '@/providers/sources/gomovies/index';
import { insertunitScraper } from '@/providers/sources/insertunit';
import { kissAsianScraper } from '@/providers/sources/kissasian/index';
import { lookmovieScraper } from '@/providers/sources/lookmovie';
import { nsbxScraper } from '@/providers/sources/nsbx';
import { remotestreamScraper } from '@/providers/sources/remotestream';
import { showboxScraper } from '@/providers/sources/showbox/index';
import { tugaflixScraper } from '@/providers/sources/tugaflix';
import { vidsrcScraper } from '@/providers/sources/vidsrc/index';
import { zoechipScraper } from '@/providers/sources/zoechip';
import { bflixScraper } from './embeds/bflix';
import { closeLoadScraper } from './embeds/closeload';
import { fileMoonScraper } from './embeds/filemoon';
import { fileMoonMp4Scraper } from './embeds/filemoon/mp4';
import { deltaScraper } from './embeds/nsbx/delta';
import { ridooScraper } from './embeds/ridoo';
import { smashyStreamOScraper } from './embeds/smashystream/opstream';
import { smashyStreamFScraper } from './embeds/smashystream/video1';
@ -38,6 +43,7 @@ import { wootlyScraper } from './embeds/wootly';
import { goojaraScraper } from './sources/goojara';
import { hdRezkaScraper } from './sources/hdrezka';
import { nepuScraper } from './sources/nepu';
import { nitesScraper } from './sources/nites';
import { primewireScraper } from './sources/primewire';
import { ridooMoviesScraper } from './sources/ridomovies';
import { smashyStreamScraper } from './sources/smashystream';
@ -56,6 +62,7 @@ export function gatherAllSources(): Array<Sourcerer> {
zoechipScraper,
vidsrcScraper,
lookmovieScraper,
nsbxScraper,
smashyStreamScraper,
ridooMoviesScraper,
vidSrcToScraper,
@ -65,7 +72,9 @@ export function gatherAllSources(): Array<Sourcerer> {
primewireScraper,
warezcdnScraper,
insertunitScraper,
nitesScraper,
soaperTvScraper,
tugaflixScraper,
];
}
@ -87,6 +96,8 @@ export function gatherAllEmbeds(): Array<Embed> {
ridooScraper,
closeLoadScraper,
fileMoonScraper,
fileMoonMp4Scraper,
deltaScraper,
vidplayScraper,
wootlyScraper,
doodScraper,
@ -98,5 +109,6 @@ export function gatherAllEmbeds(): Array<Embed> {
vTubeScraper,
warezcdnembedHlsScraper,
warezcdnembedMp4Scraper,
bflixScraper,
];
}

42
src/providers/embeds/bflix.ts Normal file
View File

@ -0,0 +1,42 @@
import { unpack } from 'unpacker';
import { makeEmbed } from '@/providers/base';
const evalCodeRegex = /eval\((.*)\)/g;
const mp4Regex = /https?:\/\/.*\.mp4/;
export const bflixScraper = makeEmbed({
id: 'bflix',
name: 'bFlix',
rank: 113,
scrape: async (ctx) => {
const mainPage = await ctx.proxiedFetcher<string>(ctx.url);
const evalCode = mainPage.match(evalCodeRegex);
if (!evalCode) throw new Error('Failed to find eval code');
const unpacked = unpack(evalCode[0]);
const file = unpacked.match(mp4Regex);
if (!file?.[0]) throw new Error('Failed to find file');
return {
stream: [
{
id: 'primary',
type: 'file',
flags: [],
captions: [],
qualities: {
unknown: {
type: 'mp4',
url: file[0],
},
},
headers: {
Referer: 'https://bflix.gs/',
},
},
],
};
},
});

2
src/providers/embeds/filemoon/index.ts
View File

@ -11,7 +11,7 @@ const fileRegex = /file:"(.*?)"/g;
export const fileMoonScraper = makeEmbed({
id: 'filemoon',
name: 'Filemoon',
rank: 400,
rank: 300,
scrape: async (ctx) => {
const embedRes = await ctx.proxiedFetcher<string>(ctx.url, {
headers: {

37
src/providers/embeds/filemoon/mp4.ts Normal file
View File

@ -0,0 +1,37 @@
import { NotFoundError } from '@/utils/errors';
import { makeEmbed } from '../../base';
import { fileMoonScraper } from './index';
export const fileMoonMp4Scraper = makeEmbed({
id: 'filemoon-mp4',
name: 'Filemoon MP4',
rank: 400,
scrape: async (ctx) => {
const result = await fileMoonScraper.scrape(ctx);
if (!result.stream) throw new NotFoundError('Failed to find result');
if (result.stream[0].type !== 'hls') throw new NotFoundError('Failed to find hls stream');
const url = result.stream[0].playlist.replace(/\/hls2\//, '/download/').replace(/\.m3u8/, '.mp4');
return {
stream: [
{
id: 'primary',
type: 'file',
qualities: {
unknown: {
type: 'mp4',
url,
},
},
flags: [],
captions: result.stream[0].captions,
},
],
};
},
});

17
src/providers/embeds/nsbx/delta.ts Normal file
View File

@ -0,0 +1,17 @@
import { EmbedOutput, makeEmbed } from '@/providers/base';
import { headers } from '@/providers/sources/nsbx';
export const deltaScraper = makeEmbed({
id: 'delta',
name: 'Delta',
rank: 200,
disabled: false,
async scrape(ctx) {
const url = `https://api.nsbx.ru/provider?resourceId=${encodeURIComponent(ctx.url)}&provider=delta`;
const result = await ctx.fetcher(url, {
headers,
});
return result as EmbedOutput;
},
});

79
src/providers/sources/nites.ts Normal file
View File

@ -0,0 +1,79 @@
import { load } from 'cheerio';
import { SourcererOutput, makeSourcerer } from '@/providers/base';
import { compareMedia } from '@/utils/compare';
import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
import { NotFoundError } from '@/utils/errors';
const baseUrl = 'https://w1.nites.is';
async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promise<SourcererOutput> {
const searchPage = await ctx.proxiedFetcher('/wp-admin/admin-ajax.php', {
baseUrl,
method: 'POST',
body: new URLSearchParams({
action: 'ajax_pagination',
query_vars: 'mixed',
search: ctx.media.title,
}),
});
const $search = load(searchPage);
const searchResults: { title: string; year: number; url: string }[] = [];
$search('li').each((_, element) => {
const title = $search(element).find('.entry-title').first().text().trim();
const year = parseInt($search(element).find('.year').first().text().trim(), 10);
const url = $search(element).find('.lnk-blk').attr('href');
if (!title || !year || !url) return;
searchResults.push({ title, year, url });
});
let watchPageUrl = searchResults.find((x) => x && compareMedia(ctx.media, x.title, x.year))?.url;
if (!watchPageUrl) throw new NotFoundError('No watchable item found');
if (ctx.media.type === 'show') {
const match = watchPageUrl.match(/\/series\/([^/]+)\/?/);
if (!match) throw new Error('Failed to parse watch page url');
watchPageUrl = watchPageUrl.replace(
`/series/${match[1]}`,
`/episode/${match[1]}-${ctx.media.season.number}x${ctx.media.episode.number}`,
);
}
const watchPage = load(await ctx.proxiedFetcher(watchPageUrl));
// it embeds vidsrc when it bflix does not has the stream
// i think all shows embed vidsrc, not sure
const embedUrl = watchPage('ul.bx-lst li a:contains("- Bflix")')
.closest('aside')
.next('div.video-options')
.find('iframe')
.attr('data-lazy-src');
if (!embedUrl) throw new Error('Failed to find embed url');
const embedPage = load(await ctx.proxiedFetcher(embedUrl));
const url = embedPage('iframe').attr('src');
if (!url) throw new Error('Failed to find embed url');
return {
embeds: [
{
embedId: 'bflix',
url,
},
],
};
}
export const nitesScraper = makeSourcerer({
id: 'nites',
name: 'Nites',
rank: 90,
flags: [],
scrapeMovie: comboScraper,
scrapeShow: comboScraper,
});

46
src/providers/sources/nsbx.ts Normal file
View File

@ -0,0 +1,46 @@
import { flags } from '@/entrypoint/utils/targets';
import { SourcererEmbed, SourcererOutput, makeSourcerer } from '@/providers/base';
import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
import { NotFoundError } from '@/utils/errors';
export const headers = {
Origin: 'https://extension.works.again.with.nsbx',
Referer: 'https://extension.works.again.with.nsbx',
};
async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promise<SourcererOutput> {
const query = {
title: ctx.media.title,
releaseYear: ctx.media.releaseYear,
tmdbId: ctx.media.tmdbId,
imdbId: ctx.media.imdbId,
type: ctx.media.type,
season: '',
episode: '',
};
if (ctx.media.type === 'show') {
query.season = ctx.media.season.number.toString();
query.episode = ctx.media.episode.number.toString();
}
const result = await ctx.fetcher(`https://api.nsbx.ru/search?query=${encodeURIComponent(JSON.stringify(query))}`, {
headers,
});
if (result.embeds.length === 0) throw new NotFoundError('No watchable item found');
return {
embeds: result.embeds as SourcererEmbed[],
};
}
export const nsbxScraper = makeSourcerer({
id: 'nsbx',
name: 'NSBX',
rank: 150,
flags: [flags.CORS_ALLOWED],
disabled: false,
scrapeMovie: comboScraper,
scrapeShow: comboScraper,
});

21
src/providers/sources/tugaflix/common.ts Normal file
View File

@ -0,0 +1,21 @@
import { load } from 'cheerio';
export const baseUrl = 'https://tugaflix.best/';
export function parseSearch(page: string): { title: string; year?: number; url: string }[] {
const results: { title: string; year?: number; url: string }[] = [];
const $ = load(page);
$('.items .poster').each((_, element) => {
const $link = $(element).find('a');
const url = $link.attr('href');
// ex title: Home Alone (1990)
const [, title, year] = $link.attr('title')?.match(/^(.*?)\s*(?:\((\d{4})\))?\s*$/) || [];
if (!title || !url) return;
// tiles dont always have the year
results.push({ title, year: year ? parseInt(year, 10) : undefined, url });
});
return results;
}

116
src/providers/sources/tugaflix/index.ts Normal file
View File

@ -0,0 +1,116 @@
import { load } from 'cheerio';
import { flags } from '@/entrypoint/utils/targets';
import { SourcererEmbed, makeSourcerer } from '@/providers/base';
import { compareMedia } from '@/utils/compare';
import { NotFoundError } from '@/utils/errors';
import { baseUrl, parseSearch } from './common';
export const tugaflixScraper = makeSourcerer({
id: 'tugaflix',
name: 'Tugaflix',
rank: 73,
flags: [flags.IP_LOCKED],
scrapeMovie: async (ctx) => {
const searchResults = parseSearch(
await ctx.proxiedFetcher<string>('/filmes/', {
baseUrl,
query: {
s: ctx.media.title,
},
}),
);
if (searchResults.length === 0) throw new NotFoundError('No watchable item found');
const url = searchResults.find((x) => x && compareMedia(ctx.media, x.title, x.year))?.url;
if (!url) throw new NotFoundError('No watchable item found');
const videoPage = await ctx.proxiedFetcher<string>(url, {
method: 'POST',
body: new URLSearchParams({ play: '' }),
});
const $ = load(videoPage);
const embeds: SourcererEmbed[] = [];
for (const element of $('.play a')) {
const embedUrl = $(element).attr('href');
if (!embedUrl) continue;
const embedPage = await ctx.proxiedFetcher.full(
embedUrl.startsWith('https://') ? embedUrl : `https://${embedUrl}`,
);
const finalUrl = load(embedPage.body)('a:contains("Download Filme")').attr('href');
if (!finalUrl) continue;
if (finalUrl.includes('streamtape')) {
embeds.push({
embedId: 'streamtape',
url: finalUrl,
});
// found doodstream on a few shows, maybe movies use it too?
// the player 2 is just streamtape in a custom player
} else if (finalUrl.includes('dood')) {
embeds.push({
embedId: 'dood',
url: finalUrl,
});
}
}
return {
embeds,
};
},
scrapeShow: async (ctx) => {
const searchResults = parseSearch(
await ctx.proxiedFetcher<string>('/series/', {
baseUrl,
query: {
s: ctx.media.title,
},
}),
);
if (searchResults.length === 0) throw new NotFoundError('No watchable item found');
const url = searchResults.find((x) => x && compareMedia(ctx.media, x.title, x.year))?.url;
if (!url) throw new NotFoundError('No watchable item found');
const s = ctx.media.season.number < 10 ? `0${ctx.media.season.number}` : ctx.media.season.number.toString();
const e = ctx.media.episode.number < 10 ? `0${ctx.media.episode.number}` : ctx.media.episode.number.toString();
const videoPage = await ctx.proxiedFetcher(url, {
method: 'POST',
body: new URLSearchParams({ [`S${s}E${e}`]: '' }),
});
const embedUrl = load(videoPage)('iframe[name="player"]').attr('src');
if (!embedUrl) throw new Error('Failed to find iframe');
const playerPage = await ctx.proxiedFetcher(embedUrl.startsWith('https:') ? embedUrl : `https:${embedUrl}`, {
method: 'POST',
body: new URLSearchParams({ submit: '' }),
});
const embeds: SourcererEmbed[] = [];
const finalUrl = load(playerPage)('a:contains("Download Episodio")').attr('href');
if (finalUrl?.includes('streamtape')) {
embeds.push({
embedId: 'streamtape',
url: finalUrl,
});
} else if (finalUrl?.includes('dood')) {
embeds.push({
embedId: 'dood',
url: finalUrl,
});
}
return {
embeds,
};
},
});

10
src/providers/sources/vidsrcto/index.ts
View File

@ -60,10 +60,16 @@ const universalScraper = async (ctx: ShowScrapeContext | MovieScrapeContext): Pr
const urlWithSubtitles = embedArr.find((v) => v.source === 'Vidplay' && v.url.includes('sub.info'))?.url;
const subtitleUrl = urlWithSubtitles ? new URL(urlWithSubtitles).searchParams.get('sub.info') : null;
if (subtitleUrl) fullUrl.searchParams.set('sub.info', subtitleUrl);
embeds.push({
embeds.push(
{
embedId: 'filemoon',
url: fullUrl.toString(),
});
},
{
embedId: 'filemoon-mp4',
url: fullUrl.toString(),
},
);
}
}