thumbnail scraping

This commit is contained in:
mrjvs 2023-10-21 04:50:14 +02:00
parent 6395d75d78
commit 32f031ab23
6 changed files with 336 additions and 34 deletions

View File

@ -1,8 +1,48 @@
import { useCallback, useEffect, useRef } from "react"; import {
MouseEvent,
RefObject,
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from "react";
import { useProgressBar } from "@/hooks/useProgressBar"; import { useProgressBar } from "@/hooks/useProgressBar";
import { nearestImageAt } from "@/stores/player/slices/thumbnails";
import { usePlayerStore } from "@/stores/player/store"; import { usePlayerStore } from "@/stores/player/store";
function ThumbnailDisplay(props: { at: number }) {
const thumbnailImages = usePlayerStore((s) => s.thumbnails.images);
const currentThumbnail = useMemo(() => {
return nearestImageAt(thumbnailImages, props.at)?.image;
}, [thumbnailImages, props.at]);
if (!currentThumbnail) return null;
return <img src={currentThumbnail.data} className="h-12" />;
}
function useMouseHoverPosition(barRef: RefObject<HTMLDivElement>) {
const [mousePos, setMousePos] = useState(-1);
const mouseMove = useCallback(
(e: MouseEvent<HTMLDivElement>) => {
const bar = barRef.current;
if (!bar) return;
const rect = barRef.current.getBoundingClientRect();
const pos = (e.pageX - rect.left) / barRef.current.offsetWidth;
setMousePos(pos * 100);
},
[setMousePos, barRef]
);
const mouseLeave = useCallback(() => {
setMousePos(-1);
}, [setMousePos]);
return { mousePos, mouseMove, mouseLeave };
}
export function ProgressBar() { export function ProgressBar() {
const { duration, time, buffered } = usePlayerStore((s) => s.progress); const { duration, time, buffered } = usePlayerStore((s) => s.progress);
const display = usePlayerStore((s) => s.display); const display = usePlayerStore((s) => s.display);
@ -18,6 +58,7 @@ export function ProgressBar() {
); );
const ref = useRef<HTMLDivElement>(null); const ref = useRef<HTMLDivElement>(null);
const { mouseMove, mouseLeave, mousePos } = useMouseHoverPosition(ref);
const { dragging, dragPercentage, dragMouseDown } = useProgressBar( const { dragging, dragPercentage, dragMouseDown } = useProgressBar(
ref, ref,
@ -31,45 +72,67 @@ export function ProgressBar() {
setDraggingTime((dragPercentage / 100) * duration); setDraggingTime((dragPercentage / 100) * duration);
}, [setDraggingTime, duration, dragPercentage]); }, [setDraggingTime, duration, dragPercentage]);
return ( const mousePosition = Math.floor(dragPercentage * duration);
<div className="w-full" ref={ref}>
<div
className="group w-full h-8 flex items-center cursor-pointer"
onMouseDown={dragMouseDown}
onTouchStart={dragMouseDown}
>
<div
className={[
"relative w-full h-1 bg-video-progress-background bg-opacity-25 rounded-full transition-[height] duration-100 group-hover:h-1.5",
dragging ? "!h-1.5" : "",
].join(" ")}
>
{/* Pre-loaded content bar */}
<div
className="absolute top-0 left-0 h-full rounded-full bg-video-progress-preloaded bg-opacity-50 flex justify-end items-center"
style={{
width: `${(buffered / duration) * 100}%`,
}}
/>
{/* Actual progress bar */} return (
<div className="w-full relative">
<div className="top-0 absolute inset-x-0">
{mousePos > -1 ? (
<div <div
className="absolute top-0 left-0 h-full rounded-full bg-video-progress-watched flex justify-end items-center" className="absolute bottom-0"
style={{ style={{
width: `${ left: `${mousePos}%`,
Math.max(
0,
Math.min(1, dragging ? dragPercentage / 100 : time / duration)
) * 100
}%`,
}} }}
> >
<ThumbnailDisplay at={mousePosition} />
</div>
) : null}
</div>
<div className="w-full" ref={ref}>
<div
className="group w-full h-8 flex items-center cursor-pointer"
onMouseDown={dragMouseDown}
onTouchStart={dragMouseDown}
onMouseLeave={mouseLeave}
onMouseMove={mouseMove}
>
<div
className={[
"relative w-full h-1 bg-video-progress-background bg-opacity-25 rounded-full transition-[height] duration-100 group-hover:h-1.5",
dragging ? "!h-1.5" : "",
].join(" ")}
>
{/* Pre-loaded content bar */}
<div <div
className={[ className="absolute top-0 left-0 h-full rounded-full bg-video-progress-preloaded bg-opacity-50 flex justify-end items-center"
"w-[1rem] min-w-[1rem] h-[1rem] rounded-full transform translate-x-1/2 scale-0 group-hover:scale-100 bg-white transition-[transform] duration-100", style={{
isSeeking ? "scale-100" : "", width: `${(buffered / duration) * 100}%`,
].join(" ")} }}
/> />
{/* Actual progress bar */}
<div
className="absolute top-0 left-0 h-full rounded-full bg-video-progress-watched flex justify-end items-center"
style={{
width: `${
Math.max(
0,
Math.min(
1,
dragging ? dragPercentage / 100 : time / duration
)
) * 100
}%`,
}}
>
<div
className={[
"w-[1rem] min-w-[1rem] h-[1rem] rounded-full transform translate-x-1/2 scale-0 group-hover:scale-100 bg-white transition-[transform] duration-100",
isSeeking ? "scale-100" : "",
].join(" ")}
/>
</div>
</div> </div>
</div> </div>
</div> </div>

View File

@ -5,6 +5,7 @@ import { CastingInternal } from "@/components/player/internals/CastingInternal";
import { HeadUpdater } from "@/components/player/internals/HeadUpdater"; import { HeadUpdater } from "@/components/player/internals/HeadUpdater";
import { KeyboardEvents } from "@/components/player/internals/KeyboardEvents"; import { KeyboardEvents } from "@/components/player/internals/KeyboardEvents";
import { ProgressSaver } from "@/components/player/internals/ProgressSaver"; import { ProgressSaver } from "@/components/player/internals/ProgressSaver";
import { ThumbnailScraper } from "@/components/player/internals/ThumbnailScraper";
import { VideoClickTarget } from "@/components/player/internals/VideoClickTarget"; import { VideoClickTarget } from "@/components/player/internals/VideoClickTarget";
import { VideoContainer } from "@/components/player/internals/VideoContainer"; import { VideoContainer } from "@/components/player/internals/VideoContainer";
import { PlayerHoverState } from "@/stores/player/slices/interface"; import { PlayerHoverState } from "@/stores/player/slices/interface";
@ -82,6 +83,7 @@ export function Container(props: PlayerProps) {
return ( return (
<div className="relative"> <div className="relative">
<BaseContainer> <BaseContainer>
<ThumbnailScraper />
<CastingInternal /> <CastingInternal />
<VideoContainer /> <VideoContainer />
<ProgressSaver /> <ProgressSaver />

View File

@ -0,0 +1,127 @@
import Hls from "hls.js";
import { useEffect, useMemo, useRef } from "react";
import { ThumbnailImage } from "@/stores/player/slices/thumbnails";
import { usePlayerStore } from "@/stores/player/store";
import { LoadableSource, selectQuality } from "@/stores/player/utils/qualities";
class ThumnbnailWorker {
interrupted: boolean;
videoEl: HTMLVideoElement | null = null;
canvasEl: HTMLCanvasElement | null = null;
hls: Hls | null = null;
cb: (img: ThumbnailImage) => void;
constructor(ops: { addImage: (img: ThumbnailImage) => void }) {
this.cb = ops.addImage;
this.interrupted = false;
}
start(source: LoadableSource) {
const el = document.createElement("video");
const canvas = document.createElement("canvas");
this.hls = new Hls();
if (source.type === "mp4") {
el.src = source.url;
el.crossOrigin = "anonymous";
} else if (source.type === "hls") {
this.hls.attachMedia(el);
this.hls.loadSource(source.url);
} else throw new Error("Invalid loadable source type");
this.videoEl = el;
this.canvasEl = canvas;
this.begin().catch((err) => console.error(err));
}
destroy() {
this.hls?.detachMedia();
this.hls?.destroy();
this.hls = null;
this.interrupted = true;
this.videoEl = null;
this.canvasEl = null;
}
private async initVideo() {
if (!this.videoEl || !this.canvasEl) return;
await new Promise((resolve, reject) => {
this.videoEl?.addEventListener("loadedmetadata", resolve);
this.videoEl?.addEventListener("error", reject);
});
if (!this.videoEl || !this.canvasEl) return;
this.canvasEl.height = this.videoEl.videoHeight;
this.canvasEl.width = this.videoEl.videoWidth;
}
private async takeSnapshot(at: number) {
if (!this.videoEl || !this.canvasEl) return;
this.videoEl.currentTime = at;
await new Promise((resolve) => {
this.videoEl?.addEventListener("seeked", resolve);
});
if (!this.videoEl || !this.canvasEl) return;
const ctx = this.canvasEl.getContext("2d");
if (!ctx) return;
ctx.drawImage(
this.videoEl,
0,
0,
this.canvasEl.width,
this.canvasEl.height
);
const imgUrl = this.canvasEl.toDataURL();
this.cb({
at,
data: imgUrl,
});
}
private async begin() {
const vid = this.videoEl;
if (!vid) return;
await this.initVideo();
if (this.interrupted) return;
await this.takeSnapshot(vid.duration / 2);
}
}
export function ThumbnailScraper() {
const addImage = usePlayerStore((s) => s.thumbnails.addImage);
const source = usePlayerStore((s) => s.source);
const workerRef = useRef<ThumnbnailWorker | null>(null);
const inputStream = useMemo(() => {
if (!source) return null;
return selectQuality(source, {
automaticQuality: false,
lastChosenQuality: "360",
});
}, [source]);
// TODO stop worker on meta change
// start worker with the stream
useEffect(() => {
// dont interrupt existing working
if (workerRef.current) return;
if (!inputStream) return;
const ins = new ThumnbnailWorker({
addImage,
});
workerRef.current = ins;
ins.start(inputStream.stream);
}, [inputStream, addImage]);
// destroy worker on unmount
useEffect(() => {
return () => {
if (workerRef.current) workerRef.current.destroy();
};
}, []);
return null;
}

View File

@ -0,0 +1,106 @@
import { MakeSlice } from "@/stores/player/slices/types";
export interface ThumbnailImage {
at: number;
data: string;
}
export interface ThumbnailSlice {
thumbnails: {
images: ThumbnailImage[];
addImage(img: ThumbnailImage): void;
};
}
export interface ThumbnailImagePosition {
index: number;
image: ThumbnailImage;
}
/**
* get nearest image at the timestamp provided
* @param images images, must be sorted
*/
export function nearestImageAt(
images: ThumbnailImage[],
at: number
): ThumbnailImagePosition | null {
// no images, early return
if (images.length === 0) return null;
const indexPastTimestamp = images.findIndex((v) => v.at < at);
// no image found past timestamp, so last image must be closest
if (indexPastTimestamp === -1)
return {
index: images.length - 1,
image: images[images.length - 1],
};
const imagePastTimestamp = images[indexPastTimestamp];
// if past timestamp is first image, just return that image
if (indexPastTimestamp === 0)
return {
index: indexPastTimestamp,
image: imagePastTimestamp,
};
// distance before distance past
// | |
// [before] --------------------- [at] --------------------- [past]
const imageBeforeTimestamp = images[indexPastTimestamp - 1];
const distanceBefore = at - imageBeforeTimestamp.at;
const distancePast = imagePastTimestamp.at - at;
// if distance of before timestamp is smaller than the distance past
// before is closer, return that
// [before] --X-------------- [past]
if (distanceBefore < distancePast)
return {
index: indexPastTimestamp - 1,
image: imageBeforeTimestamp,
};
// must be closer to past here, return past
// [before] --------------X-- [past]
return {
index: indexPastTimestamp,
image: imagePastTimestamp,
};
}
export const createThumbnailSlice: MakeSlice<ThumbnailSlice> = (set, get) => ({
thumbnails: {
images: [],
addImage(img) {
const store = get();
const exactOrPastImageIndex = store.thumbnails.images.findIndex(
(v) => v.at <= img.at
);
// not found past or exact, so just append to the end
if (exactOrPastImageIndex === -1) {
set((s) => {
s.thumbnails.images.push(img);
});
return;
}
const exactOrPastImage = store.thumbnails.images[exactOrPastImageIndex];
// found exact, replace data
if (exactOrPastImage.at === img.at) {
set((s) => {
s.thumbnails.images[exactOrPastImageIndex] = img;
});
return;
}
// found one past, insert right before it
set((s) => {
s.thumbnails.images.splice(exactOrPastImageIndex, 0, img);
});
},
},
});

View File

@ -6,13 +6,15 @@ import { InterfaceSlice } from "@/stores/player/slices/interface";
import { PlayingSlice } from "@/stores/player/slices/playing"; import { PlayingSlice } from "@/stores/player/slices/playing";
import { ProgressSlice } from "@/stores/player/slices/progress"; import { ProgressSlice } from "@/stores/player/slices/progress";
import { SourceSlice } from "@/stores/player/slices/source"; import { SourceSlice } from "@/stores/player/slices/source";
import { ThumbnailSlice } from "@/stores/player/slices/thumbnails";
export type AllSlices = InterfaceSlice & export type AllSlices = InterfaceSlice &
PlayingSlice & PlayingSlice &
ProgressSlice & ProgressSlice &
SourceSlice & SourceSlice &
DisplaySlice & DisplaySlice &
CastingSlice; CastingSlice &
ThumbnailSlice;
export type MakeSlice<Slice> = StateCreator< export type MakeSlice<Slice> = StateCreator<
AllSlices, AllSlices,
[["zustand/immer", never]], [["zustand/immer", never]],

View File

@ -7,6 +7,7 @@ import { createInterfaceSlice } from "@/stores/player/slices/interface";
import { createPlayingSlice } from "@/stores/player/slices/playing"; import { createPlayingSlice } from "@/stores/player/slices/playing";
import { createProgressSlice } from "@/stores/player/slices/progress"; import { createProgressSlice } from "@/stores/player/slices/progress";
import { createSourceSlice } from "@/stores/player/slices/source"; import { createSourceSlice } from "@/stores/player/slices/source";
import { createThumbnailSlice } from "@/stores/player/slices/thumbnails";
import { AllSlices } from "@/stores/player/slices/types"; import { AllSlices } from "@/stores/player/slices/types";
export const usePlayerStore = create( export const usePlayerStore = create(
@ -17,5 +18,6 @@ export const usePlayerStore = create(
...createSourceSlice(...a), ...createSourceSlice(...a),
...createDisplaySlice(...a), ...createDisplaySlice(...a),
...createCastingSlice(...a), ...createCastingSlice(...a),
...createThumbnailSlice(...a),
})) }))
); );