This commit is contained in:
mrjvs 2023-09-27 19:00:40 +02:00
parent 78507a24c8
commit de63979e4d
17 changed files with 428 additions and 45 deletions

View File

@ -0,0 +1,3 @@
code > span {
white-space: pre;
}

View File

@ -18,44 +18,36 @@ Let's get started with `@movie-web/providers`. First lets install the package.
To get started with scraping on the **server**, first you have to make an instance of the providers.
::code-group
```ts [index.ts (server)]
import { makeProviders, makeDefaultFetcher, targets } from '@movie-web/providers';
```ts
import { makeProviders, makeDefaultFetcher, targets } from '@movie-web/providers';
// this is how the library will make http requests
const myFetcher = makeDefaultFetcher(fetch);
// this is how the library will make http requests
const myFetcher = makeDefaultFetcher(fetch);
// make an instance of the providers library
const providers = makeProviders({
fetcher: myFetcher,
// make an instance of the providers library
const providers = makeProviders({
fetcher: myFetcher,
// will be played on a native video player
target: targets.NATIVE
})
```
::
// will be played on a native video player
target: targets.NATIVE
})
```
Perfect, now we can start scraping a stream:
::code-group
```ts [index.ts (server)]
// fetch some data from TMDB
const media = {
type: 'movie',
title: "Hamilton",
releaseYear: 2020,
tmdbId: "556574"
}
const output = await providers.runAll({
media: media
})
```ts [index.ts (server)]
// fetch some data from TMDB
const media = {
type: 'movie',
title: "Hamilton",
releaseYear: 2020,
tmdbId: "556574"
}
const output = await providers.runAll({
media: media
})
if (!output) console.log("No stream found")
console.log(`stream url: ${output.stream.playlist}`)
```
::
::alert{type="info"}
Thats it! Next up: [check out some examples](/examples).
::
if (!output) console.log("No stream found")
console.log(`stream url: ${output.stream.playlist}`)
```

View File

@ -0,0 +1,13 @@
# Targets
When making an instance of the library using `makeProviders()`. It will immediately require choosing a target.
::alert{type="info"}
A target is the device where the stream will be played on.
**Where the scraping is run has nothing to do with the target**, only where the stream is finally played in the end is significant in choosing a target.
::
#### Possible targets
- **`targets.BROWSER`** Stream will be played in a browser with CORS
- **`targets.NATIVE`** Stream will be played natively
- **`targets.ALL`** Stream will be played on a device with no restrictions of any kind

View File

@ -0,0 +1,47 @@
# Fetchers
When making an instance of the library using `makeProviders()`. It will immediately make a fetcher.
This comes with some considerations depending on the environment youre running.
## Using `fetch()`
In most cases, you can use the `fetch()` API. This will work in newer versions of node (18 and above) and on the browser.
```ts
const fetcher = makeDefaultFetcher(fetch);
```
If you using older version of nodejs. You can use the npm package `node-fetch` to polyfill fetch:
```ts
import fetch from "node-fetch";
const fetcher = makeDefaultFetcher(fetch);
```
## Using fetchers on the browser
when using this library on a browser, you will need a proxy. Browser come with a ton of restrictions on when a webrequest can be made, and to bypass those restrictions you will need a cors proxy.
The movie-web team has a proxy pre-made and pre-configured for you to use. for more information you can check out [movie-web/simple-proxy](https://github.com/movie-web/simple-proxy). After installing you can use this proxy like so:
```ts
const fetcher = makeSimpleProxyFetcher("https://your.proxy.workers.dev/", fetch);
```
If you aren't able to use this specific proxy and need to use a different one, you can make your own fetcher in the next section.
## Making a custom fetcher
In some rare cases, a custom fetcher will need to be made. This can be quite difficult to do from scratch so it's recommended to base it off an existing fetcher and building your own functionality around it.
```ts
export function makeCustomFetcher(): Fetcher {
const fetcher = makeStandardFetcher(f);
const customFetcher: Fetcher = (url, ops) => {
return fetcher(url, ops);
};
return customFetcher;
}
```
If you need to make your own fetcher for a proxy. Make sure you make it compatible with the following headers: `Cookie`, `Referer`, `Origin`. Proxied fetchers need to be able to write those headers when making a request.

View File

@ -1,2 +1,2 @@
icon: ph:star-duotone
icon: ph:book-open-fill
navigation.redirect: /guide/usage

View File

@ -0,0 +1,34 @@
# `makeProviders`
Make an instance of providers with configuration.
This is the main entrypoint of the library. It is recommended to make one instance globally and reuse it throughout your application.
## Example
```ts
import { targets, makeProviders, makeDefaultFetcher } from "@movie-web/providers";
const providers = makeProviders({
fetcher: makeDefaultFetcher(fetch),
target: targets.NATIVE, // target native app streams
});
```
## Type
```ts
function makeProviders(ops: ProviderBuilderOptions): ProviderControls;
interface ProviderBuilderOptions {
// instance of a fetcher, all webrequests are made with the fetcher.
fetcher: Fetcher;
// instance of a fetcher, in case the request has cors restrictions.
// this fetcher will be called instead of normal fetcher.
// if your environment doesnt have cors restrictions (like nodejs), there is no need to set this.
proxiedFetcher?: Fetcher;
// target to get streams for
target: Targets;
}
```

View File

@ -0,0 +1,61 @@
# `ProviderControls.runAll`
Run all providers one by one in order of their builtin ranking.
You can attach events if you need to know what is going on while its processing.
## Example
```ts
// media from TMDB
const media = {
type: 'movie',
title: "Hamilton",
releaseYear: 2020,
tmdbId: "556574"
}
// scrape a stream
const stream = await providers.runAll({
media: media,
})
// scrape a stream, but prioritize flixhq above all
// (other scrapers are stil ran if flixhq fails, it just has priority)
const flixhqStream = await providers.runAll({
media: media,
sourceOrder: ['flixhq']
})
```
## Type
```ts
function runAll(runnerOps: RunnerOptions): Promise<RunOutput | null>;
interface RunnerOptions {
// overwrite the order of sources to run. list of ids
// any omitted ids are in added to the end in order of rank (highest first)
sourceOrder?: string[];
// overwrite the order of embeds to run. list of ids
// any omitted ids are in added to the end in order of rank (highest first)
embedOrder?: string[];
// object of event functions
events?: FullScraperEvents;
// the media you want to see sources from
media: ScrapeMedia;
}
type RunOutput = {
// source scraper id
sourceId: string;
// if from an embed, this is the embed scraper id
embedId?: string;
// the outputed stream
stream: Stream;
};
```

View File

@ -0,0 +1,66 @@
# `ProviderControls.runSourceScraper`
Run a specific source scraper and get its outputted streams.
## Example
```ts
import { SourcererOutput, NotFoundError } from "@movie-web/providers";
// media from TMDB
const media = {
type: 'movie',
title: "Hamilton",
releaseYear: 2020,
tmdbId: "556574"
}
// scrape a stream from flixhq
let output: SourcererOutput;
try {
output = await providers.runSourceScraper({
id: 'flixhq',
media: media,
})
} catch (err) {
if (err instanceof NotFoundError) {
console.log("source doesnt have this media");
} else {
console.log("failed to scrape")
}
return;
}
if (!output.stream && output.embeds.length === 0) {
console.log("no streams found");
}
```
## Type
```ts
function runSourceScraper(runnerOps: SourceRunnerOptions): Promise<SourcererOutput>;
interface SourceRunnerOptions {
// object of event functions
events?: IndividualScraperEvents;
// the media you want to see sources from
media: ScrapeMedia;
// id of the source scraper you want to scrape from
id: string;
}
type SourcererOutput = {
// list of embeds that the source scraper found.
// embed id is a reference to an embed scraper
embeds: {
embedId: string;
url: string;
}[];
// the stream that the scraper found
stream?: Stream;
};
```

View File

@ -0,0 +1,44 @@
# `ProviderControls.runEmbedScraper`
Run a specific embed scraper and get its outputted streams.
## Example
```ts
import { SourcererOutput } from "@movie-web/providers";
// scrape a stream from upcloud
let output: EmbedOutput;
try {
output = await providers.runSourceScraper({
id: 'upcloud',
url: 'https://example.com/123',
})
} catch (err) {
console.log("failed to scrape")
return;
}
// output.stream now has your stream
```
## Type
```ts
function runEmbedScraper(runnerOps: SourceRunnerOptions): Promise<EmbedOutput>;
interface EmbedRunnerOptions {
// object of event functions
events?: IndividualScraperEvents;
// the embed url
url: string;
// id of the embed scraper you want to scrape from
id: string;
}
type EmbedOutput = {
stream: Stream;
};
```

View File

@ -0,0 +1,25 @@
# `ProviderControls.listSources`
List all source scrapers that applicable for the target.
They are sorted by rank, highest first
## Example
```ts
const sourceScrapers = providers.listSources();
// garuanteed to only return type: 'source'
```
## Type
```ts
function listSources(): MetaOutput[];
type MetaOutput = {
type: 'embed' | 'source';
id: string;
rank: number;
name: string;
mediaTypes?: Array<MediaTypes>;
};
```

View File

@ -0,0 +1,25 @@
# `ProviderControls.listEmbeds`
List all embed scrapers that applicable for the target.
They are sorted by rank, highest first
## Example
```ts
const embedScrapers = providers.listEmbeds();
// garuanteed to only return type: 'embed'
```
## Type
```ts
function listEmbeds(): MetaOutput[];
type MetaOutput = {
type: 'embed' | 'source';
id: string;
rank: number;
name: string;
mediaTypes?: Array<MediaTypes>;
};
```

View File

@ -0,0 +1,24 @@
# `ProviderControls.getMetadata`
Get meta data for a scraper, can be either source or embed scraper.
Returns null if id is not recognized.
## Example
```ts
const flixhqSource = providers.getMetadata('flixhq');
```
## Type
```ts
function getMetadata(id: string): MetaOutput | null;
type MetaOutput = {
type: 'embed' | 'source';
id: string;
rank: number;
name: string;
mediaTypes?: Array<MediaTypes>;
};
```

View File

@ -0,0 +1,20 @@
# `makeStandardFetcher`
Make a fetcher from a `fetch()` API. It is used for making a instance of providers with `makeProviders()`.
## Example
```ts
import { targets, makeProviders, makeDefaultFetcher } from "@movie-web/providers";
const providers = makeProviders({
fetcher: makeDefaultFetcher(fetch),
target: targets.NATIVE,
});
```
## Type
```ts
function makeDefaultFetcher(fetchApi: typeof fetch): Fetcher;
```

View File

@ -0,0 +1,23 @@
# `makeSimpleProxyFetcher`
Make a fetcher to use with [movie-web/simple-proxy](https://github.com/movie-web/simple-proxy). This is for making a proxiedFetcher, so you can run this library in the browser.
## Example
```ts
import { targets, makeProviders, makeDefaultFetcher, makeSimpleProxyFetcher } from "@movie-web/providers";
const proxyUrl = "https://your.proxy.workers.dev/"
const providers = makeProviders({
fetcher: makeDefaultFetcher(fetch),
proxiedFetcher: makeSimpleProxyFetcher(proxyUrl, fetch),
target: targets.BROWSER,
});
```
## Type
```ts
function makeSimpleProxyFetcher(proxyUrl: string, fetchApi: typeof fetch): Fetcher;
```

View File

@ -0,0 +1,2 @@
icon: ph:file-code-fill
navigation.redirect: /api/makeproviders

View File

@ -2,6 +2,10 @@ export default defineNuxtConfig({
// https://github.com/nuxt-themes/docus
extends: '@nuxt-themes/docus',
css: [
'@/assets/css/main.css',
],
modules: [
// https://github.com/nuxt-modules/plausible
'@nuxtjs/plausible',

View File

@ -3,16 +3,16 @@ import { defineTheme } from 'pinceau'
export default defineTheme({
color: {
primary: {
50: "#FFF6E5",
100: "#FFEDCC",
200: "#FFDB99",
300: "#FFC966",
400: "#FFB833",
500: "#FFA500",
600: "#CC8500",
700: "#996300",
800: "#664200",
900: "#332100"
50: "#F5E5FF",
100: "#E7CCFF",
200: "#D4A9FF",
300: "#BE85FF",
400: "#A861FF",
500: "#8E3DFF",
600: "#7F36D4",
700: "#662CA6",
800: "#552578",
900: "#441E49"
}
}
})