new backend interfaces

This commit is contained in:
Jelle van Snik 2023-01-11 21:16:48 +01:00
parent 8268abc45d
commit f1257973e7
12 changed files with 203 additions and 12 deletions

View File

@ -0,0 +1,20 @@
import { MWEmbedType } from "../helpers/embed";
import { registerEmbedScraper } from "../helpers/register";
import { MWStreamType } from "../helpers/streams";
registerEmbedScraper({
id: "testembed",
rank: 23,
for: MWEmbedType.OPENLOAD,
async getStream({ progress, url }) {
console.log("scraping url: ", url);
progress(25);
progress(50);
progress(75);
return {
streamUrl: "hello-world",
type: MWStreamType.MP4,
};
},
});

View File

@ -0,0 +1,24 @@
import { MWStream } from "./streams";
export enum MWEmbedType {
OPENLOAD = "openload",
}
export type MWEmbed = {
type: MWEmbedType | null;
url: string;
};
export type MWEmbedContext = {
progress(percentage: number): void;
url: string;
};
export type MWEmbedScraper = {
id: string;
for: MWEmbedType;
rank: number;
disabled?: boolean;
getStream(ctx: MWEmbedContext): Promise<MWStream>;
};

View File

@ -0,0 +1,23 @@
import { MWMediaType } from "../metadata/types";
import { MWEmbed } from "./embed";
import { MWStream } from "./streams";
export type MWProviderScrapeResult = {
stream?: MWStream;
embeds: MWEmbed[];
};
export type MWProviderContext = {
progress(percentage: number): void;
imdbId: string;
tmdbId: string;
};
export type MWProvider = {
id: string;
rank: number;
disabled?: boolean;
type: MWMediaType[];
scrape(ctx: MWProviderContext): Promise<MWProviderScrapeResult>;
};

View File

@ -0,0 +1,61 @@
import { MWEmbedScraper } from "./embed";
import { MWProvider } from "./provider";
let providers: MWProvider[] = [];
let embeds: MWEmbedScraper[] = [];
export function registerProvider(provider: MWProvider) {
if (provider.disabled) return;
providers.push(provider);
}
export function registerEmbedScraper(embed: MWEmbedScraper) {
if (embed.disabled) return;
embeds.push(embed);
}
export function initializeScraperStore() {
// sort by ranking
providers = providers.sort((a, b) => a.rank - b.rank);
embeds = embeds.sort((a, b) => a.rank - b.rank);
// check for invalid ranks
let lastRank: null | number = null;
providers.forEach((v) => {
if (lastRank === null) {
lastRank = v.rank;
return;
}
if (lastRank === v.rank)
throw new Error(`Duplicate rank number for provider ${v.id}`);
lastRank = v.rank;
});
lastRank = null;
providers.forEach((v) => {
if (lastRank === null) {
lastRank = v.rank;
return;
}
if (lastRank === v.rank)
throw new Error(`Duplicate rank number for embed scraper ${v.id}`);
lastRank = v.rank;
});
// check for duplicate ids
const providerIds = providers.map((v) => v.id);
if (
providerIds.length > 0 &&
new Set(providerIds).size !== providerIds.length
)
throw new Error("Duplicate IDS in providers");
const embedIds = embeds.map((v) => v.id);
if (embedIds.length > 0 && new Set(embedIds).size !== embedIds.length)
throw new Error("Duplicate IDS in embed scrapers");
}
export function getProviders(): MWProvider[] {
return providers;
}
export function getEmbeds(): MWEmbedScraper[] {
return embeds;
}

View File

@ -0,0 +1,9 @@
export enum MWStreamType {
MP4 = "mp4",
HLS = "hls",
}
export type MWStream = {
streamUrl: string;
type: MWStreamType;
};

17
src/backend/index.ts Normal file
View File

@ -0,0 +1,17 @@
import { initializeScraperStore } from "./helpers/register";
// TODO backend system:
// - run providers/embedscrapers in webworkers for multithreading and isolation
// - caption support
// - hooks to run all providers one by one
// - move over old providers to new system
// - implement jons providers/embedscrapers
// providers
// -- nothing here yet
import "./providers/testProvider";
// embeds
// -- nothing here yet
initializeScraperStore();

View File

@ -1,4 +1,5 @@
import { MWMediaType, MWQuery } from "@/providers"; import { MWMediaType, MWQuery } from "@/providers";
import { MWMediaMeta } from "./types";
const JW_API_BASE = "https://apis.justwatch.com"; const JW_API_BASE = "https://apis.justwatch.com";
@ -27,18 +28,10 @@ type JWPage<T> = {
total_results: number; total_results: number;
}; };
export type MWSearchResult = {
title: string;
id: string;
year: string;
poster?: string;
type: MWMediaType;
};
export async function searchForMedia({ export async function searchForMedia({
searchQuery, searchQuery,
type, type,
}: MWQuery): Promise<MWSearchResult[]> { }: MWQuery): Promise<MWMediaMeta[]> {
const body: JWSearchQuery = { const body: JWSearchQuery = {
content_types: [], content_types: [],
page: 1, page: 1,
@ -56,7 +49,7 @@ export async function searchForMedia({
)}` )}`
).then((res) => res.json() as Promise<JWPage<JWSearchResults>>); ).then((res) => res.json() as Promise<JWPage<JWSearchResults>>);
return data.items.map<MWSearchResult>((v) => ({ return data.items.map<MWMediaMeta>((v) => ({
title: v.title, title: v.title,
id: v.id.toString(), id: v.id.toString(),
year: v.original_release_year.toString(), year: v.original_release_year.toString(),

View File

@ -0,0 +1,13 @@
export enum MWMediaType {
MOVIE = "movie",
SERIES = "series",
ANIME = "anime",
}
export type MWMediaMeta = {
title: string;
id: string;
year: string;
poster?: string;
type: MWMediaType;
};

View File

@ -1 +0,0 @@
this folder will be used for provider helper methods and the like

View File

@ -1 +0,0 @@
the new list of all providers, the old ones will go and be rewritten

View File

@ -0,0 +1,32 @@
import { MWEmbedType } from "../helpers/embed";
import { registerProvider } from "../helpers/register";
import { MWStreamType } from "../helpers/streams";
import { MWMediaType } from "../metadata/types";
registerProvider({
id: "testprov",
rank: 42,
type: [MWMediaType.MOVIE],
async scrape({ progress, imdbId, tmdbId }) {
console.log("scraping provider for: ", imdbId, tmdbId);
progress(25);
progress(50);
progress(75);
// providers can optionally provide a stream themselves,
// incase they host their own streams instead of using embeds
return {
stream: {
streamUrl: "hello-world",
type: MWStreamType.HLS,
},
embeds: [
{
type: MWEmbedType.OPENLOAD,
url: "https://google.com",
},
],
};
},
});

View File

@ -7,6 +7,7 @@ import { conf } from "@/setup/config";
import App from "@/setup/App"; import App from "@/setup/App";
import "@/setup/i18n"; import "@/setup/i18n";
import "@/setup/index.css"; import "@/setup/index.css";
import "@/backend";
// initialize // initialize
const key = const key =