Merge branch 'dev' into pr-14-v2

.docs/content/0.index.md

@@ -8,7 +8,7 @@ layout: page
 ---
 cta:
   - Get Started
-  - /guide/usage
+  - /get-started/introduction
 secondary:
   - Open on GitHub →
   - https://github.com/movie-web/providers

.docs/content/1.Guide/1.targets.md

@@ -1,13 +0,0 @@
-# Targets
-
-When making an instance of the library using `makeProviders()`. It will immediately require choosing a target.
-
-::alert{type="info"}
-A target is the device where the stream will be played on.
-**Where the scraping is run has nothing to do with the target**, only where the stream is finally played in the end is significant in choosing a target.
-::
-
-#### Possible targets
-- **`targets.BROWSER`** Stream will be played in a browser with CORS
-- **`targets.NATIVE`** Stream will be played natively
-- **`targets.ALL`** Stream will be played on a device with no restrictions of any kind

.docs/content/1.Guide/_dir.yml

@@ -1,2 +0,0 @@
-icon: ph:book-open-fill
-navigation.redirect: /guide/usage

.docs/content/1.get-started/0.introduction.md

@@ -0,0 +1,14 @@
+# Introduction
+
+## What is `@movie-web/providers`?
+
+`@movie-web/providers` is the soul of [movie-web.app](https://movie-web.app). It's a collection of scrapers of various streaming sites. It extracts the raw streams from those sites, so you can watch them without any extra fluff from the original sites.
+
+## What can I use this on?
+
+We support many different environments, here are a few examples:
+ - In browser, watch streams without needing a server to scrape (does need a proxy)
+ - In a native app, scrape in the app itself
+ - In a backend server, scrape on the server and give the streams to the client to watch.
+
+To find out how to configure the library for your environment, You can read [How to use on X](../2.essentials/0.usage-on-x.md).

.docs/content/1.get-started/1.quick-start.md

@@ -1,4 +1,6 @@
-# Usage
+# Quick start
+
+## Installation
 
 Let's get started with `@movie-web/providers`. First lets install the package.
 
@@ -18,11 +20,15 @@ Let's get started with `@movie-web/providers`. First lets install the package.
 
 To get started with scraping on the **server**, first you have to make an instance of the providers.
 
-```ts
+::alert{type="warning"}
-import { makeProviders, makeDefaultFetcher, targets } from '@movie-web/providers';
+This snippet will only work on a **server**, for other environments, check out [Usage on X](../2.essentials/0.usage-on-x.md).
+::
+
+```ts [index.ts (server)]
+import { makeProviders, makeStandardFetcher, targets } from '@movie-web/providers';
 
 // this is how the library will make http requests
-const myFetcher = makeDefaultFetcher(fetch);
+const myFetcher = makeStandardFetcher(fetch);
 
 // make an instance of the providers library
 const providers = makeProviders({
@@ -33,7 +39,8 @@ const providers = makeProviders({
 })
 ```
 
-Perfect, now we can start scraping a stream:
+Perfect, this instance of the providers you can reuse everywhere where you need to.
+Now lets actually scrape an item:
 
 ```ts [index.ts (server)]
 // fetch some data from TMDB
@@ -47,7 +54,7 @@ const media = {
 const output = await providers.runAll({
   media: media
 })
-
-if (!output) console.log("No stream found")
-console.log(`stream url: ${output.stream.playlist}`)
 ```
+
+Now we have our stream in the output variable. (If the output is `null` then nothing could be found.)
+To find out how to use the streams, check out [Using streams](../2.essentials/4.using-streams.md).

.docs/content/1.get-started/3.examples.md

@@ -0,0 +1,5 @@
+# Examples
+
+::alert{type="warning"}
+There are no examples yet, stay tuned!
+::

.docs/content/1.get-started/4.changelog.md

@@ -0,0 +1,34 @@
+---
+title: 'Changelog'
+---
+
+# Version 2.0.1
+- Fixed issue where febbox-mp4 would not show all qualities
+- Fixed issue where discoverEmbeds event would not show the embeds in the right order
+
+# Version 2.0.0
+
+::alert{type="warning"}
+There are breaking changes in this list, make sure to read them thoroughly if you plan on updating.
+::
+
+**Development tooling:**
+- Added integration test for browser. To make sure the package keeps working in the browser
+- Add type checking when building, previously it ignored them
+- Refactored the main folder, now called entrypoint.
+- Dev-cli code has been split up a bit more, a bit cleaner to navigate
+- Dev-cli is now moved to `npm run cli`
+- Dev-cli has now has support for running in a headless browser using a proxy URL.
+- Fetchers can now return a full response with headers and everything
+
+**New features:**
+- Added system to allow scraping ip locked sources through the consistentIpforRequests option.
+- There is now a `buildProviders()` function that gives a builder for the `ProviderControls`. It's an alternative to `makeProviders()`.
+- Streams can now return a headers object and a `preferredHeaders` object. which is required and optional headers for when using the stream.
+
+**Notable changes:**
+- Renamed the NO_CORS flag to CORS_ALLOWED (meaning that resource sharing is allowed)
+- Export Fetcher and Stream types with all types related to it
+- Providers can now return a list of streams instead of just one.
+- Captions now have identifiers returned with them. Just generally useful to have
+- New targets and some of them renamed

.docs/content/1.get-started/_dir.yml

@@ -0,0 +1,2 @@
+icon: ph:shooting-star-fill
+navigation.redirect: /get-started/introduction

.docs/content/2.Api/7.makeStandardFetcher.md

@@ -1,20 +0,0 @@
-# `makeStandardFetcher`
-
-Make a fetcher from a `fetch()` API. It is used for making a instance of providers with `makeProviders()`.
-
-## Example
-
-```ts
-import { targets, makeProviders, makeDefaultFetcher } from "@movie-web/providers";
-
-const providers = makeProviders({
-  fetcher: makeDefaultFetcher(fetch),
-  target: targets.NATIVE,
-});
-```
-
-## Type
-
-```ts
-function makeDefaultFetcher(fetchApi: typeof fetch): Fetcher;
-```

.docs/content/2.Api/_dir.yml

@@ -1,2 +0,0 @@
-icon: ph:file-code-fill
-navigation.redirect: /api/makeproviders

.docs/content/2.essentials/0.usage-on-x.md

@@ -0,0 +1,50 @@
+# How to use on X
+
+The library can run in many environments, so it can be tricky to figure out how to set it up.
+
+So here is a checklist, for more specific environments, keep reading below:
+ - When requests are very restricted (like browser client-side). Configure a proxied fetcher.
+ - When your requests come from the same device it will be streamed on (Not compatible with proxied fetcher). Set `consistentIpForRequests: true`.
+ - To set a target. Consult [Targets](./1.targets.md).
+
+To make use of the examples below, You check check out the following pages:
+ - [Quick start](../1.get-started/1.quick-start.md)
+ - [Using streams](../2.essentials/4.using-streams.md)
+
+## NodeJs server
+```ts
+import { makeProviders, makeStandardFetcher, targets } from '@movie-web/providers';
+
+const providers = makeProviders({
+  fetcher: makeStandardFetcher(fetch),
+  target: chooseYourself, // check out https://providers.docs.movie-web.app/essentials/targets
+})
+```
+
+## Browser client-side
+
+Using the provider package client-side requires a hosted version of simple-proxy.
+Read more [about proxy fetchers](./2.fetchers.md#using-fetchers-on-the-browser).
+
+```ts
+import { makeProviders, makeStandardFetcher, targets } from '@movie-web/providers';
+
+const proxyUrl = "https://your.proxy.workers.dev/";
+
+const providers = makeProviders({
+  fetcher: makeStandardFetcher(fetch),
+  proxiedFetcher: makeSimpleProxyFetcher(proxyUrl, fetch),
+  target: target.BROWSER,
+})
+```
+
+## React native
+```ts
+import { makeProviders, makeStandardFetcher, targets } from '@movie-web/providers';
+
+const providers = makeProviders({
+  fetcher: makeStandardFetcher(fetch),
+  target: target.NATIVE,
+  consistentIpForRequests: true,
+})
+```

.docs/content/2.essentials/1.targets.md

@@ -0,0 +1,14 @@
+# Targets
+
+When creating provider controls, you will immediately be required to choose a target.
+
+::alert{type="warning"}
+A target is the device where the stream will be played on.
+**Where the scraping is run has nothing to do with the target**, only where the stream is finally played in the end is significant in choosing a target.
+::
+
+#### Possible targets
+- **`targets.BROWSER`** Stream will be played in a browser with CORS
+- **`targets.BROWSER_EXTENSION`** Stream will be played in a browser using the movie-web extension (WIP)
+- **`targets.NATIVE`** Stream will be played on a native video player
+- **`targets.ANY`** No restrictions for selecting streams, will just give all of them

.docs/content/2.essentials/2.fetchers.md

@@ -1,13 +1,13 @@
 # Fetchers
 
-When making an instance of the library using `makeProviders()`. It will immediately make a fetcher.
+When creating provider controls, it will need you to configure a fetcher.
 This comes with some considerations depending on the environment youre running.
 
 ## Using `fetch()`
 In most cases, you can use the `fetch()` API. This will work in newer versions of Node.js (18 and above) and on the browser.
 
 ```ts
-const fetcher = makeDefaultFetcher(fetch);
+const fetcher = makeStandardFetcher(fetch);
 ```
 
 If you using older version of Node.js. You can use the npm package `node-fetch` to polyfill fetch:
@@ -15,7 +15,7 @@ If you using older version of Node.js. You can use the npm package `node-fetch`
 ```ts
 import fetch from "node-fetch";
 
-const fetcher = makeDefaultFetcher(fetch);
+const fetcher = makeStandardFetcher(fetch);
 ```
 
 ## Using fetchers on the browser
@@ -29,7 +29,7 @@ const fetcher = makeSimpleProxyFetcher("https://your.proxy.workers.dev/", fetch)
 
 If you aren't able to use this specific proxy and need to use a different one, you can make your own fetcher in the next section.
 
-## Making a custom fetcher
+## Making a derived fetcher
 
 In some rare cases, a custom fetcher will need to be made. This can be quite difficult to do from scratch so it's recommended to base it off an existing fetcher and building your own functionality around it.
 
@@ -37,6 +37,7 @@ In some rare cases, a custom fetcher will need to be made. This can be quite dif
 export function makeCustomFetcher(): Fetcher {
   const fetcher = makeStandardFetcher(f);
   const customFetcher: Fetcher = (url, ops) => {
+    // Do something with the options and url here
     return fetcher(url, ops);
   };
 
@@ -44,4 +45,30 @@ export function makeCustomFetcher(): Fetcher {
 }
 ```
 
-If you need to make your own fetcher for a proxy. Make sure you make it compatible with the following headers: `Cookie`, `Referer`, `Origin`. Proxied fetchers need to be able to write those headers when making a request.
+If you need to make your own fetcher for a proxy. Make sure you make it compatible with the following headers: `Set-Cookie`, `Cookie`, `Referer`, `Origin`. Proxied fetchers need to be able to write/read those headers when making a request.
+
+
+## Making a fetcher from scratch
+
+In some even rare cases, you need to make one completely from scratch.
+This is the list of features it needs:
+ - Send/read every header
+ - Parse JSON, otherwise parse as text
+ - Send JSON, Formdata or normal strings
+ - get final destination url
+
+It's not recommended to do this at all, but if you have to. You can base your code on the original implementation of `makeStandardFetcher`. Check the out [source code for it here](https://github.com/movie-web/providers/blob/dev/src/fetchers/standardFetch.ts).
+
+Here is a basic template on how to make your own custom fetcher:
+
+```ts
+const myFetcher: Fetcher = (url, ops) => {
+  // Do some fetching
+  return {
+    body: {},
+    finalUrl: '',
+    headers: new Headers(), // should only contain headers from ops.readHeaders
+    statusCode: 200,
+  };
+}
+```

.docs/content/2.essentials/3.customize-providers.md

@@ -0,0 +1,74 @@
+# Customize providers
+
+You make a provider controls in two ways. Either with `makeProviders()` (the simpler option) or with `buildProviders()` (more elaborate and extensive option).
+
+## `makeProviders()` (simple)
+
+To know what to set the configuration to, you can read [How to use on X](./0.usage-on-x.md) for a detailed guide on how to configure your controls.
+
+```ts
+const providers = makeProviders({
+  // fetcher, every web request gets called through here
+  fetcher: makeStandardFetcher(fetch),
+  
+  // proxied fetcher, if the scraper needs to access a CORS proxy. this fetcher will be called instead
+  // of the normal fetcher. Defaults to the normal fetcher.
+  proxiedFetcher: undefined;
+
+  // target of where the streams will be used
+  target: targets.NATIVE;
+
+  // Set this to true, if the requests will have the same IP as
+  // the device that the stream will be played on.
+  consistentIpForRequests: false;
+})
+
+```
+
+## `buildProviders()` (advanced)
+
+To know what to set the configuration to, you can read [How to use on X](./0.usage-on-x.md) for a detailed guide on how to configure your controls.
+
+### Standard setup
+
+```ts
+const providers = buildProviders()
+  .setTarget(targets.NATIVE) // target of where the streams will be used
+  .setFetcher(makeStandardFetcher(fetch)) // fetcher, every web request gets called through here
+  .addBuiltinProviders() // add all builtin providers, if this is not called, no providers will be added to the controls
+  .build();
+```
+
+### Adding only select few providers
+
+Not all providers are great quality, so you can make a instance of the controls with only the providers you want.
+
+```ts
+const providers = buildProviders()
+  .setTarget(targets.NATIVE) // target of where the streams will be used
+  .setFetcher(makeStandardFetcher(fetch)) // fetcher, every web request gets called through here
+  .addSource('showbox') // only add showbox source
+  .addEmbed('febbox-hls') // add febbox-hls embed, which is returned by showbox
+  .build();
+```
+
+
+### Adding your own scrapers to the providers
+
+If you have your own scraper and still want to use the nice utils of the provider library or just want to add on to the builtin providers. You can add your own custom source.
+
+```ts
+const providers = buildProviders()
+  .setTarget(targets.NATIVE) // target of where the streams will be used
+  .setFetcher(makeStandardFetcher(fetch)) // fetcher, every web request gets called through here
+  .addSource({ // add your own source
+    id: 'my-scraper',
+    name: 'My scraper',
+    rank: 800,
+    flags: [],
+    scrapeMovie(ctx) {
+      throw new Error('Not implemented');
+    }
+  })
+  .build();
+```

.docs/content/2.essentials/4.using-streams.md

@@ -0,0 +1,84 @@
+# Using streams
+
+Streams can sometimes be quite picky on how they can be used. So here is a guide on how to use them.
+
+## Essentials
+
+All streams have the same common parameters:
+ - `Stream.type`: The type of stream. Either `hls` or `file`
+ - `Stream.id`: The id of this stream, unique per scraper output.
+ - `Stream.flags`: A list of flags that apply to this stream. Most people won't need to use it.
+ - `Stream.captions`: A list of captions/subtitles for this stream.
+ - `Stream.headers`: Either undefined or a key value object of headers you must set to use the stream.
+ - `Stream.preferredHeaders`: Either undefined or a key value object of headers you may want to set if you want optimal playback - but not required.
+
+Now let's delve deeper into how to actually watch these streams!
+
+## Streams with type `hls`
+
+HLS streams can be tough to watch, it's not a normal file you can just use.
+These streams have an extra property `Stream.playlist` which contains the m3u8 playlist.
+
+Here is a code sample of how to use HLS streams in web context using hls.js
+
+```html
+<script src="https://cdn.jsdelivr.net/npm/hls.js@1"></script>
+
+<video id="video"></video>
+<script>
+  const stream = null; // add your stream here
+
+  if (Hls.isSupported()) {
+    var video = document.getElementById('video');
+    var hls = new Hls();
+    hls.loadSource(stream.playlist);
+    hls.attachMedia(video);
+  }
+</script>
+```
+
+## Streams with type `file`
+
+File streams are quite easy to use, it just returns a new property: `Stream.qualities`.
+This property is a map of quality and a stream file. So if you want to get 1080p quality you do `stream["1080"]` to get your stream file. It will return undefined if there is no quality like that.
+
+The possibly qualities are: `unknown`, `360`, `480`, `720`, `1080`, `4k`.
+File based streams are garuanteed to always have one quality.
+
+Once you get a streamfile, you have the following parameters:
+ - `StreamFile.type`: Right now it can only be `mp4`.
+ - `StreamFile.url`: The URL linking to the video file.
+
+Here is a code sample of how to watch a file based stream the video in a browser:
+
+```html
+<video id="video"></video>
+<script>
+  const stream = null; // add your stream here
+  const video = document.getElementById('video');
+
+  const qualityEntries = Object.keys(stream.qualities);
+  const firstQuality = qualityEntries[0];
+  video.src = firstQuality.url;
+</script>
+```
+
+## Streams with headers
+
+Streams have both a `Stream.headers` and a `Stream.preferredHeaders`.
+The difference between the two is that `Stream.headers` **must** be set in other for the stream to work. While the other one is optional, and can only enhance the quality or performance.
+
+If your target is set to `BROWSER`. There will never be required headers, as it's not possible to do.
+
+## Using captions/subtitles
+
+All streams have a list of captions at `Stream.captions`. The structure looks like this:
+```ts
+type Caption = {
+  type: CaptionType; // Language type, either "srt" or "vtt"
+  id: string; // Unique per stream
+  url: string; // The url pointing to the subtitle file
+  hasCorsRestrictions: boolean; // If true, you will need to proxy it if you're running in a browser
+  language: string; // Language code of the caption
+};
+```

.docs/content/2.essentials/_dir.yml

@@ -0,0 +1,3 @@
+icon: ph:info-fill
+navigation.redirect: /essentials/usage
+navigation.title: "Get started"

.docs/content/3.in-depth/0.sources-and-embeds.md

@@ -0,0 +1,11 @@
+# Sources vs embeds
+
+::alert{type="warning"}
+This page isn't quite done yet, stay tuned!
+::
+
+<!--
+TODO
+- How do sources and embeds differ
+- How do sources and embeds interact
+-->

.docs/content/3.in-depth/1.new-providers.md

@@ -0,0 +1,12 @@
+# New providers
+
+::alert{type="warning"}
+This page isn't quite done yet, stay tuned!
+::
+
+<!--
+TODO
+- How to make new sources or embeds
+- Ranking
+- Link to flags
+-->

.docs/content/3.in-depth/2.flags.md

@@ -0,0 +1,10 @@
+# Flags
+
+Flags is the primary way the library seperates entities between different environments.
+For example some sources only give back content that has the CORS headers set to allow anyone, so that source gets the flag `CORS_ALLOWED`. Now if you set your target to `BROWSER`, sources without that flag won't even get listed.
+
+This concept is applied in multiple away across the library.
+
+## Flag options
+ - `CORS_ALLOWED`: Headers from the output streams are set to allow any origin.
+ - `IP_LOCKED`: The streams are locked by ip, requester and watcher must be the same.

.docs/content/3.in-depth/_dir.yml

@@ -0,0 +1,3 @@
+icon: ph:atom-fill
+navigation.redirect: /in-depth/sources-and-embeds
+navigation.title: "In-depth"

.docs/content/4.extra-topics/0.development.md

@@ -0,0 +1,72 @@
+# Development / contributing
+
+::alert{type="warning"}
+This page isn't quite done yet, stay tuned!
+::
+
+<!--
+TODO
+- Development setup
+- How to make new sources/embeds (link to the page)
+- How to use the fetchers, when to use proxiedFetcher
+- How to use the context
+-->
+
+## Testing using the CLI
+
+Testing can be quite difficult for this library, unit tests can't really be made because of the unreliable nature of scrapers.
+But manually testing by writing an entrypoint is also really annoying.
+
+Our solution is to make a CLI that you can use to run the scrapers, for everything else there are unit tests.
+
+### Setup
+Make a `.env` file in the root of the repository and add a TMDB api key: `MOVIE_WEB_TMDB_API_KEY=KEY_HERE`.
+Then make sure you've ran `npm i` to get all the dependencies.
+
+### Mode 1 - interactive
+
+To run the CLI without needing to learn all the arguments, simply run the following command and go with the flow.
+
+```sh
+npm run cli
+```
+
+### Mode 2 - arguments
+
+For repeatability, it can be useful to specify the arguments one by one.
+To see all the arguments, you can run the help command:
+```sh
+npm run cli -- -h
+```
+
+Then just run it with your arguments, for example:
+```sh
+npm run cli -- -sid showbox -tid 556574
+```
+
+### Examples
+
+```sh
+# Spirited away - showbox
+npm run cli -- -sid showbox -tid 129
+
+# Hamilton - flixhq
+npm run cli -- -sid flixhq -tid 556574
+
+# Arcane S1E1 - showbox
+npm run cli -- -sid zoechip -tid 94605 -s 1 -e 1
+
+# febbox mp4 - get streams from an embed (gotten from a source output)
+npm run cli -- -sid febbox-mp4 -u URL_HERE
+```
+
+### Fetcher options
+
+The CLI comes with a few built-in fetchers:
+ - `node-fetch`: Fetch using the "node-fetch" library.
+ - `native`: Use the new fetch built into Node.JS (undici).
+ - `browser`: Start up headless chrome, and run the library in that context using a proxied fetcher.
+
+::alert{type="warning"}
+The browser fetcher will require you to run `npm run build` before running the CLI. Otherwise you will get outdated results.
+::

.docs/content/4.extra-topics/_dir.yml

@@ -0,0 +1,3 @@
+icon: ph:aperture-fill
+navigation.redirect: /extra-topics/development
+navigation.title: "Extra topics"

.docs/content/5.api-reference/0.makeProviders.md

@@ -1,12 +1,12 @@
 # `makeProviders`
 
-Make an instance of providers with configuration. 
+Make an instance of provider controls with configuration. 
 This is the main entrypoint of the library. It is recommended to make one instance globally and reuse it throughout your application.
 
 ## Example
 
 ```ts
-import { targets, makeProviders, makeDefaultFetcher } from "@movie-web/providers";
+import { targets, makeProviders, makeDefaultFetcher } from '@movie-web/providers';
 
 const providers = makeProviders({
   fetcher: makeDefaultFetcher(fetch),
@@ -25,7 +25,7 @@ interface ProviderBuilderOptions {
   
   // instance of a fetcher, in case the request has cors restrictions.
   // this fetcher will be called instead of normal fetcher.
-  // if your environment doesnt have cors restrictions (like nodejs), there is no need to set this.
+  // if your environment doesnt have cors restrictions (like Node.JS), there is no need to set this.
   proxiedFetcher?: Fetcher;
 
   // target to get streams for

.docs/content/5.api-reference/1.ProviderControlsRunAll.md

@@ -9,9 +9,9 @@ You can attach events if you need to know what is going on while its processing.
 // media from TMDB
 const media = {
   type: 'movie',
-  title: "Hamilton",
+  title: 'Hamilton',
   releaseYear: 2020,
-  tmdbId: "556574"
+  tmdbId: '556574'
 }
 
 // scrape a stream

.docs/content/5.api-reference/2.ProviderControlsrunSourceScraper.md

@@ -5,14 +5,14 @@ Run a specific source scraper and get its outputted streams.
 ## Example
 
 ```ts
-import { SourcererOutput, NotFoundError } from "@movie-web/providers";
+import { SourcererOutput, NotFoundError } from '@movie-web/providers';
 
 // media from TMDB
 const media = {
   type: 'movie',
-  title: "Hamilton",
+  title: 'Hamilton',
   releaseYear: 2020,
-  tmdbId: "556574"
+  tmdbId: '556574'
 }
 
 // scrape a stream from flixhq
@@ -24,15 +24,15 @@ try {
   })
 } catch (err) {
   if (err instanceof NotFoundError) {
-    console.log("source doesnt have this media");
+    console.log('source doesnt have this media');
   } else {
-    console.log("failed to scrape")
+    console.log('failed to scrape')
   }
   return;
 }
 
 if (!output.stream && output.embeds.length === 0) {
-  console.log("no streams found");
+  console.log('no streams found');
 }
 ```
 

.docs/content/5.api-reference/3.ProviderControlsrunEmbedScraper.md

@@ -5,7 +5,7 @@ Run a specific embed scraper and get its outputted streams.
 ## Example
 
 ```ts
-import { SourcererOutput } from "@movie-web/providers";
+import { SourcererOutput } from '@movie-web/providers';
 
 // scrape a stream from upcloud
 let output: EmbedOutput;
@@ -15,7 +15,7 @@ try {
     url: 'https://example.com/123',
   })
 } catch (err) {
-  console.log("failed to scrape")
+  console.log('failed to scrape')
   return;
 }
 

.docs/content/5.api-reference/7.makeStandardFetcher.md

@@ -0,0 +1,20 @@
+# `makeStandardFetcher`
+
+Make a fetcher from a `fetch()` API. It is used for making a instance of provider controls.
+
+## Example
+
+```ts
+import { targets, makeProviders, makeDefaultFetcher } from '@movie-web/providers';
+
+const providers = makeProviders({
+  fetcher: makeStandardFetcher(fetch),
+  target: targets.ANY,
+});
+```
+
+## Type
+
+```ts
+function makeStandardFetcher(fetchApi: typeof fetch): Fetcher;
+```

.docs/content/5.api-reference/8.makeSimpleProxyFetcher.md

@@ -5,9 +5,9 @@ Make a fetcher to use with [movie-web/simple-proxy](https://github.com/movie-web
 ## Example
 
 ```ts
-import { targets, makeProviders, makeDefaultFetcher, makeSimpleProxyFetcher } from "@movie-web/providers";
+import { targets, makeProviders, makeDefaultFetcher, makeSimpleProxyFetcher } from '@movie-web/providers';
 
-const proxyUrl = "https://your.proxy.workers.dev/"
+const proxyUrl = 'https://your.proxy.workers.dev/'
 
 const providers = makeProviders({
   fetcher: makeDefaultFetcher(fetch),

.docs/content/5.api-reference/_dir.yml

@@ -0,0 +1,3 @@
+icon: ph:code-simple-fill
+navigation.redirect: /api/makeproviders
+navigation.title: "Api reference"

.github/CODEOWNERS

@@ -1,3 +1 @@
-* @movie-web/core
+* @movie-web/project-leads
-
-.github @binaryoverload

README.md

@@ -9,27 +9,6 @@ features:
 
 Visit documentation here: https://providers.docs.movie-web.app/
 
-## Development
+## How to run locally or test my changes
-To make testing scrapers easier during development a CLI tool is available to run specific sources. To run the CLI testing tool, use `npm run cli`. The script supports 2 execution modes
 
-- CLI Mode, for passing in arguments directly to the script
+These topics are also covered in the documentation, [read about it here](https://providers.docs.movie-web.app/extra-topics/development).
-- Question Mode, where the script asks you questions about which source you wish to test
-
-The following CLI Mode arguments are available
-
-| Argument      | Alias  | Description                                                             | Default      |
-|---------------|--------|-------------------------------------------------------------------------|--------------|
-| `--fetcher`   | `-f`   | Fetcher type. Either `node-fetch` or `native`                           | `node-fetch` |
-| `--source-id` | `-sid` | Source ID for the source to be tested                                   |              |
-| `--tmdb-id`   | `-tid` | TMDB ID for the media to scrape. Only used if source is a provider      |              |
-| `--type`      | `-t`   | Media type. Either `movie` or `show`. Only used if source is a provider | `movie`      |
-| `--season`    | `-s`   | Season number. Only used if type is `show`                              | `0`          |
-| `--episode`   | `-e`   | Episode number. Only used if type is `show`                             | `0`          |
-| `--url`       | `-u`   | URL to a video embed. Only used if source is an embed                   |              |
-| `--help`      | `-h`   | Shows help for the command arguments                                    |              |
-
-Example testing the FlixHQ source on the movie "Spirited Away"
-
-```bash
-npm run cli -- -sid flixhq -tid 129 -t movie
-```

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@movie-web/providers",
-  "version": "1.1.5",
+  "version": "2.0.3",
   "description": "Package that contains all the providers of movie-web",
   "main": "./lib/index.umd.js",
   "types": "./lib/index.d.ts",

src/dev-cli/validate.ts

@@ -81,6 +81,7 @@ export async function processOptions(sources: Array<Embed | Sourcerer>, options:
   const providerOptions: ProviderMakerOptions = {
     fetcher,
     target: targets.ANY,
+    consistentIpForRequests: true,
   };
 
   return {

src/entrypoint/providers.ts

@@ -2,9 +2,9 @@ import { gatherAllEmbeds, gatherAllSources } from '@/providers/all';
 import { Embed, Sourcerer } from '@/providers/base';
 
 export function getBuiltinSources(): Sourcerer[] {
-  return gatherAllSources();
+  return gatherAllSources().filter((v) => !v.disabled);
 }
 
 export function getBuiltinEmbeds(): Embed[] {
-  return gatherAllEmbeds();
+  return gatherAllEmbeds().filter((v) => !v.disabled);
 }

src/index.ts

@@ -1,6 +1,6 @@
 export type { EmbedOutput, SourcererOutput } from '@/providers/base';
 export type { Stream, StreamFile, FileBasedStream, HlsBasedStream, Qualities } from '@/providers/streams';
-export type { Fetcher, FetcherOptions, FetcherResponse } from '@/fetchers/types';
+export type { Fetcher, DefaultedFetcherOptions, FetcherOptions, FetcherResponse } from '@/fetchers/types';
 export type { RunOutput } from '@/runners/runner';
 export type { MetaOutput } from '@/entrypoint/utils/meta';
 export type { FullScraperEvents } from '@/entrypoint/utils/events';
@@ -9,6 +9,8 @@ export type { MediaTypes, ShowMedia, ScrapeMedia, MovieMedia } from '@/entrypoin
 export type { ProviderControls, RunnerOptions, EmbedRunnerOptions, SourceRunnerOptions } from '@/entrypoint/controls';
 export type { ProviderBuilder } from '@/entrypoint/builder';
 export type { ProviderMakerOptions } from '@/entrypoint/declare';
+export type { MovieScrapeContext, ShowScrapeContext, EmbedScrapeContext, ScrapeContext } from '@/utils/context';
+export type { SourcererOptions, EmbedOptions } from '@/providers/base';
 
 export { NotFoundError } from '@/utils/errors';
 export { makeProviders } from '@/entrypoint/declare';

src/providers/embeds/febbox/hls.ts

@@ -1,5 +1,4 @@
 import { MediaTypes } from '@/entrypoint/utils/media';
-import { flags } from '@/entrypoint/utils/targets';
 import { makeEmbed } from '@/providers/base';
 import { parseInputUrl } from '@/providers/embeds/febbox/common';
 import { getStreams } from '@/providers/embeds/febbox/fileList';
@@ -16,6 +15,7 @@ export const febboxHlsScraper = makeEmbed({
   id: 'febbox-hls',
   name: 'Febbox (HLS)',
   rank: 160,
+  disabled: true,
   async scrape(ctx) {
     const { type, id, season, episode } = parseInputUrl(ctx.url);
     const sharelinkResult = await ctx.proxiedFetcher<{
@@ -40,7 +40,7 @@ export const febboxHlsScraper = makeEmbed({
         {
           id: 'primary',
           type: 'hls',
-          flags: [flags.CORS_ALLOWED],
+          flags: [],
           captions: await getSubtitles(ctx, id, firstStream.fid, type as MediaTypes, season, episode),
           playlist: `https://www.febbox.com/hls/main/${firstStream.oss_fid}.m3u8`,
         },

src/providers/embeds/febbox/qualities.ts

@@ -4,24 +4,35 @@ import { ScrapeContext } from '@/utils/context';
 
 const allowedQualities = ['360', '480', '720', '1080', '4k'];
 
-export async function getStreamQualities(ctx: ScrapeContext, apiQuery: object) {
+interface FebboxQuality {
-  const mediaRes: { list: { path: string; quality: string; fid?: number }[] } = (await sendRequest(ctx, apiQuery)).data;
+  path: string;
+  real_quality: string;
+  fid?: number;
+}
 
-  const qualityMap = mediaRes.list
+function mapToQuality(quality: FebboxQuality): FebboxQuality | null {
-    .filter((file) => allowedQualities.includes(file.quality.replace('p', '')))
+  const q = quality.real_quality.replace('p', '').toLowerCase();
-    .map((file) => ({
+  if (!allowedQualities.includes(q)) return null;
-      url: file.path,
+  return {
-      quality: file.quality.replace('p', ''),
+    real_quality: q,
-    }));
+    path: quality.path,
+    fid: quality.fid,
+  };
+}
+
+export async function getStreamQualities(ctx: ScrapeContext, apiQuery: object) {
+  const mediaRes: { list: FebboxQuality[] } = (await sendRequest(ctx, apiQuery)).data;
+
+  const qualityMap = mediaRes.list.map((v) => mapToQuality(v)).filter((v): v is FebboxQuality => !!v);
 
   const qualities: Record<string, StreamFile> = {};
 
   allowedQualities.forEach((quality) => {
-    const foundQuality = qualityMap.find((q) => q.quality === quality);
+    const foundQuality = qualityMap.find((q) => q.real_quality === quality && q.path);
-    if (foundQuality && foundQuality.url) {
+    if (foundQuality) {
       qualities[quality] = {
         type: 'mp4',
-        url: foundQuality.url,
+        url: foundQuality.path,
       };
     }
   });

src/providers/embeds/febbox/subtitles.ts

@@ -37,22 +37,28 @@ export async function getSubtitles(
   const subResult = (await sendRequest(ctx, subtitleApiQuery)) as CaptionApiResponse;
   const subtitleList = subResult.data.list;
   const output: Caption[] = [];
+  const languagesAdded: Record<string, true> = {};
 
   subtitleList.forEach((sub) => {
     const subtitle = sub.subtitles.sort((a, b) => b.order - a.order)[0];
     if (!subtitle) return;
+
     const subtitleFilePath = subtitle.file_path
       .replace(captionsDomains[0], captionsDomains[1])
       .replace(/\s/g, '+')
       .replace(/[()]/g, (c) => {
         return `%${c.charCodeAt(0).toString(16)}`;
       });
+
     const subtitleType = getCaptionTypeFromUrl(subtitleFilePath);
     if (!subtitleType) return;
 
     const validCode = isValidLanguageCode(subtitle.lang);
     if (!validCode) return;
 
+    if (languagesAdded[subtitle.lang]) return;
+    languagesAdded[subtitle.lang] = true;
+
     output.push({
       id: subtitleFilePath,
       language: subtitle.lang,

src/providers/sources/lookmovie/index.ts

@@ -10,8 +10,8 @@ async function universalScraper(ctx: MovieScrapeContext | ShowScrapeContext): Pr
   if (!lookmovieData) throw new NotFoundError('Media not found');
 
   ctx.progress(30);
-  const videoUrl = await scrape(ctx, ctx.media, lookmovieData);
+  const video = await scrape(ctx, ctx.media, lookmovieData);
-  if (!videoUrl) throw new NotFoundError('No video found');
+  if (!video.playlist) throw new NotFoundError('No video found');
 
   ctx.progress(60);
 
@@ -20,10 +20,10 @@ async function universalScraper(ctx: MovieScrapeContext | ShowScrapeContext): Pr
     stream: [
       {
         id: 'primary',
-        playlist: videoUrl,
+        playlist: video.playlist,
         type: 'hls',
         flags: [flags.IP_LOCKED],
-        captions: [],
+        captions: video.captions,
       },
     ],
   };
@@ -33,6 +33,7 @@ export const lookmovieScraper = makeSourcerer({
   id: 'lookmovie',
   name: 'LookMovie',
   rank: 1,
+  disabled: true,
   flags: [flags.IP_LOCKED],
   scrapeShow: universalScraper,
   scrapeMovie: universalScraper,

src/providers/sources/lookmovie/type.ts

@@ -39,8 +39,17 @@ interface VideoSources {
   [key: string]: string;
 }
 
+interface VideoSubtitles {
+  id?: number;
+  id_movie?: number;
+  url: string;
+  language: string;
+  shard?: string;
+}
+
 export interface StreamsDataResult {
   streams: VideoSources;
+  subtitles: VideoSubtitles[];
 }
 
 export interface ResultItem {

src/providers/sources/lookmovie/util.ts

@@ -4,7 +4,9 @@ import { ScrapeContext } from '@/utils/context';
 import { NotFoundError } from '@/utils/errors';
 
 import { Result, ResultItem, ShowDataResult, episodeObj } from './type';
-import { getVideoUrl } from './video';
+import { getVideo } from './video';
+
+export const baseUrl = 'https://lmscript.xyz';
 
 export async function searchAndFindMedia(
   ctx: ScrapeContext,
@@ -12,7 +14,7 @@ export async function searchAndFindMedia(
 ): Promise<ResultItem | undefined> {
   if (media.type === 'show') {
     const searchRes = await ctx.fetcher<Result>(`/v1/shows`, {
-      baseUrl: 'https://lmscript.xyz',
+      baseUrl,
       query: { 'filters[q]': media.title },
     });
 
@@ -23,7 +25,7 @@ export async function searchAndFindMedia(
   }
   if (media.type === 'movie') {
     const searchRes = await ctx.fetcher<Result>(`/v1/movies`, {
-      baseUrl: 'https://lmscript.xyz',
+      baseUrl,
       query: { 'filters[q]': media.title },
     });
 
@@ -40,7 +42,7 @@ export async function scrape(ctx: ScrapeContext, media: MovieMedia | ShowMedia,
     id = result.id_movie;
   } else if (media.type === 'show') {
     const data = await ctx.fetcher<ShowDataResult>(`/v1/shows`, {
-      baseUrl: 'https://lmscript.xyz',
+      baseUrl,
       query: { expand: 'episodes', id: result.id_show },
     });
 
@@ -54,6 +56,6 @@ export async function scrape(ctx: ScrapeContext, media: MovieMedia | ShowMedia,
   // Check ID
   if (id === null) throw new NotFoundError('Not found');
 
-  const videoUrl = await getVideoUrl(ctx, id, media);
+  const video = await getVideo(ctx, id, media);
-  return videoUrl;
+  return video;
 }

src/providers/sources/lookmovie/video.ts

@@ -1,7 +1,9 @@
 import { MovieMedia, ShowMedia } from '@/entrypoint/utils/media';
+import { Caption } from '@/providers/captions';
 import { ScrapeContext } from '@/utils/context';
 
 import { StreamsDataResult } from './type';
+import { baseUrl } from './util';
 
 export async function getVideoSources(
   ctx: ScrapeContext,
@@ -17,17 +19,17 @@ export async function getVideoSources(
     path = `/v1/movies/view`;
   }
   const data = await ctx.fetcher<StreamsDataResult>(path, {
-    baseUrl: 'https://lmscript.xyz',
+    baseUrl,
-    query: { expand: 'streams', id },
+    query: { expand: 'streams,subtitles', id },
   });
   return data;
 }
 
-export async function getVideoUrl(
+export async function getVideo(
   ctx: ScrapeContext,
   id: string,
   media: MovieMedia | ShowMedia,
-): Promise<string | null> {
+): Promise<{ playlist: string | null; captions: Caption[] }> {
   // Get sources
   const data = await getVideoSources(ctx, id, media);
   const videoSources = data.streams;
@@ -42,5 +44,16 @@ export async function getVideoUrl(
     }
   }
 
-  return videoUrl;
+  const captions: Caption[] = data.subtitles.map((sub) => ({
+    id: sub.url,
+    type: 'vtt',
+    url: `${baseUrl}${sub.url}`,
+    hasCorsRestrictions: false,
+    language: sub.language,
+  }));
+
+  return {
+    playlist: videoUrl,
+    captions,
+  };
 }

src/providers/sources/showbox/index.ts

@@ -1,6 +1,5 @@
 import { flags } from '@/entrypoint/utils/targets';
 import { SourcererOutput, makeSourcerer } from '@/providers/base';
-import { febboxHlsScraper } from '@/providers/embeds/febbox/hls';
 import { febboxMp4Scraper } from '@/providers/embeds/febbox/mp4';
 import { compareTitle } from '@/utils/compare';
 import { MovieScrapeContext, ShowScrapeContext } from '@/utils/context';
@@ -31,10 +30,6 @@ async function comboScraper(ctx: ShowScrapeContext | MovieScrapeContext): Promis
 
   return {
     embeds: [
-      {
-        embedId: febboxHlsScraper.id,
-        url: `/${ctx.media.type}/${id}/${season}/${episode}`,
-      },
       {
         embedId: febboxMp4Scraper.id,
         url: `/${ctx.media.type}/${id}/${season}/${episode}`,

src/providers/sources/zoechip/common.ts

@@ -38,7 +38,7 @@ export async function formatSource(
         embed.embedId = mixdropScraper.id;
         break;
       default:
-        throw new Error(`Failed to find ZoeChip embed source for ${link}`);
+        return null;
     }
 
     return embed;

src/providers/streams.ts

@@ -4,7 +4,6 @@ import { Caption } from '@/providers/captions';
 export type StreamFile = {
   type: 'mp4';
   url: string;
-  headers?: Record<string, string>;
 };
 
 export type Qualities = 'unknown' | '360' | '480' | '720' | '1080' | '4k';

src/runners/runner.ts

@@ -116,9 +116,12 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
       };
     }
 
-    if (output.embeds.length > 0) {
+    // run embed scrapers on listed embeds
+    const sortedEmbeds = output.embeds.sort((a, b) => embedIds.indexOf(a.embedId) - embedIds.indexOf(b.embedId));
+
+    if (sortedEmbeds.length > 0) {
       ops.events?.discoverEmbeds?.({
-        embeds: output.embeds.map((v, i) => ({
+        embeds: sortedEmbeds.map((v, i) => ({
           id: [s.id, i].join('-'),
           embedScraperId: v.embedId,
         })),
@@ -126,10 +129,6 @@ export async function runAllProviders(list: ProviderList, ops: ProviderRunnerOpt
       });
     }
 
-    // run embed scrapers on listed embeds
-    const sortedEmbeds = output.embeds;
-    sortedEmbeds.sort((a, b) => embedIds.indexOf(a.embedId) - embedIds.indexOf(b.embedId));
-
     for (const ind in sortedEmbeds) {
       if (!Object.prototype.hasOwnProperty.call(sortedEmbeds, ind)) continue;
       const e = sortedEmbeds[ind];