import React, {useCallback, useEffect, useMemo, useRef, useState} from 'react' import Typography from '@mui/material/Typography' import Container from '@mui/material/Container' import ToggleButton from '@mui/material/ToggleButton' import ToggleButtonGroup from '@mui/material/ToggleButtonGroup' import {PB} from 'smartknobjs-proto' import {VideoInfo} from './types' import {Button, CardActions, Paper} from '@mui/material' import {exhaustiveCheck, findNClosest, lerp, NoUndefinedField} from './util' import {groupBy, parseInt} from 'lodash' import _ from 'lodash' import {SmartKnobWebSerial} from 'smartknobjs-webserial' const MIN_ZOOM = 0.01 const MAX_ZOOM = 60 const PIXELS_PER_POSITION = 5 enum Mode { Scroll = 'SKDEMO_Scroll', Frames = 'SKDEMO_Frames', Speed = 'SKDEMO_Speed', } type PlaybackState = { speed: number currentFrame: number } type InterfaceState = { zoomTimelinePixelsPerFrame: number } type Config = NoUndefinedField & { zoomTimelinePixelsPerFrame: number } export type AppProps = { info: VideoInfo } export const App: React.FC = ({info}) => { const [smartKnob, setSmartKnob] = useState(null) const [smartKnobState, setSmartKnobState] = useState>( PB.SmartKnobState.toObject(PB.SmartKnobState.create({config: PB.SmartKnobConfig.create()}), { defaults: true, }) as NoUndefinedField, ) const [smartKnobConfig, setSmartKnobConfig] = useState({ position: 0, subPositionUnit: 0, positionNonce: Math.floor(Math.random() * 255), minPosition: 0, maxPosition: 0, positionWidthRadians: (15 * Math.PI) / 180, detentStrengthUnit: 0, endstopStrengthUnit: 1, snapPoint: 0.7, text: Mode.Scroll, detentPositions: [], snapPointBias: 0, ledHue: 0, zoomTimelinePixelsPerFrame: 0.1, }) useEffect(() => { console.log('send config', smartKnobConfig) smartKnob?.sendConfig(PB.SmartKnobConfig.create(smartKnobConfig)) }, [ smartKnob, smartKnobConfig.position, smartKnobConfig.subPositionUnit, smartKnobConfig.positionNonce, smartKnobConfig.minPosition, smartKnobConfig.maxPosition, smartKnobConfig.positionWidthRadians, smartKnobConfig.detentStrengthUnit, smartKnobConfig.endstopStrengthUnit, smartKnobConfig.snapPoint, smartKnobConfig.text, smartKnobConfig.detentPositions, smartKnobConfig.snapPointBias, smartKnobConfig.ledHue, ]) const [playbackState, setPlaybackState] = useState({ speed: 0, currentFrame: 0, }) const [interfaceState, setInterfaceState] = useState({ zoomTimelinePixelsPerFrame: 0.3, }) const totalPositions = Math.ceil( (info.totalFrames * smartKnobConfig.zoomTimelinePixelsPerFrame) / PIXELS_PER_POSITION, ) const detentPositions = useMemo(() => { // Always include the first and last positions at detents const positionsToFrames = groupBy([0, ...info.boundaryFrames, info.totalFrames - 1], (frame) => Math.round((frame * smartKnobConfig.zoomTimelinePixelsPerFrame) / PIXELS_PER_POSITION), ) console.log(JSON.stringify(positionsToFrames)) return positionsToFrames }, [info.boundaryFrames, info.totalFrames, totalPositions, smartKnobConfig.zoomTimelinePixelsPerFrame]) const scrollPositionWholeMemo = useMemo(() => { const position = (playbackState.currentFrame * smartKnobConfig.zoomTimelinePixelsPerFrame) / PIXELS_PER_POSITION return Math.round(position) }, [playbackState.currentFrame, smartKnobConfig.zoomTimelinePixelsPerFrame]) const [nClosest, setNClosest] = useState>([]) useEffect(() => { const calculated = findNClosest(Object.keys(detentPositions).map(parseInt), scrollPositionWholeMemo, 5).sort( (a, b) => a - b, ) setNClosest((cur) => { if (_.isEqual(cur, calculated)) { return cur } return calculated }) }, [scrollPositionWholeMemo]) const changeMode = useCallback( (newMode: Mode) => { if (newMode === Mode.Scroll) { setSmartKnobConfig((curConfig) => { const position = (playbackState.currentFrame * curConfig.zoomTimelinePixelsPerFrame) / PIXELS_PER_POSITION const positionWhole = Math.round(position) const subPositionUnit = position - positionWhole return { position: positionWhole, subPositionUnit, positionNonce: (curConfig.positionNonce + 1) % 256, minPosition: 0, maxPosition: Math.round( ((info.totalFrames - 1) * curConfig.zoomTimelinePixelsPerFrame) / PIXELS_PER_POSITION, ), positionWidthRadians: (8 * Math.PI) / 180, detentStrengthUnit: 3, endstopStrengthUnit: 1, snapPoint: 0.7, text: Mode.Scroll, detentPositions: findNClosest(Object.keys(detentPositions).map(parseInt), position, 5), snapPointBias: 0, ledHue: 0, zoomTimelinePixelsPerFrame: curConfig.zoomTimelinePixelsPerFrame, } }) } else if (newMode === Mode.Frames) { setSmartKnobConfig((curConfig) => { return { position: Math.floor(playbackState.currentFrame), subPositionUnit: 0, positionNonce: (curConfig.positionNonce + 1) % 256, minPosition: 0, maxPosition: info.totalFrames - 1, positionWidthRadians: (1.8 * Math.PI) / 180, detentStrengthUnit: 1, endstopStrengthUnit: 1, snapPoint: 1.1, text: Mode.Frames, detentPositions: [], snapPointBias: 0, ledHue: (120 * 255) / 360, zoomTimelinePixelsPerFrame: curConfig.zoomTimelinePixelsPerFrame, } }) } else if (newMode === Mode.Speed) { setSmartKnobConfig((curConfig) => { return { position: 0, subPositionUnit: 0, positionNonce: (curConfig.positionNonce + 1) % 256, minPosition: playbackState.currentFrame === 0 ? 0 : -6, maxPosition: playbackState.currentFrame === info.totalFrames - 1 ? 0 : 6, positionWidthRadians: (30 * Math.PI) / 180, detentStrengthUnit: 1, endstopStrengthUnit: 1, snapPoint: 0.5, text: Mode.Speed, detentPositions: [], snapPointBias: 0.4, ledHue: (240 * 255) / 360, zoomTimelinePixelsPerFrame: curConfig.zoomTimelinePixelsPerFrame, } }) } else { exhaustiveCheck(newMode) } }, [detentPositions, info.totalFrames, playbackState], ) const nextMode = useCallback(() => { const curMode = smartKnobConfig.text as unknown as Mode console.log('nextMode', curMode) if (curMode === Mode.Scroll) { changeMode(Mode.Frames) } else if (curMode === Mode.Frames) { changeMode(Mode.Speed) } else if (curMode === Mode.Speed) { changeMode(Mode.Scroll) } else { exhaustiveCheck(curMode) } }, [smartKnobConfig.text, changeMode]) // Initialize to Scroll mode useEffect(() => { changeMode(Mode.Scroll) }, []) useEffect(() => { if (smartKnobState.config.text === '') { console.debug('No valid state yet') return } const currentMode = smartKnobState.config.text as Mode if (currentMode !== smartKnobConfig.text) { console.debug('Mode mismatch, ignoring state', {configMode: smartKnobConfig.text, stateMode: currentMode}) return } // Update playbackState if (currentMode === Mode.Scroll) { // TODO: round input based on zoom level to avoid noise const rawFrame = Math.trunc( ((smartKnobState.currentPosition + smartKnobState.subPositionUnit) * PIXELS_PER_POSITION) / smartKnobConfig.zoomTimelinePixelsPerFrame, ) const frame = detentPositions[smartKnobState.currentPosition]?.[0] ?? Math.min(Math.max(rawFrame, 0), info.totalFrames - 1) setPlaybackState({ speed: 0, currentFrame: frame, }) // Update config with N nearest detents setSmartKnobConfig((curConfig) => { let positionInfo: Partial = {} if (interfaceState.zoomTimelinePixelsPerFrame !== curConfig.zoomTimelinePixelsPerFrame) { const position = (playbackState.currentFrame * interfaceState.zoomTimelinePixelsPerFrame) / PIXELS_PER_POSITION const positionWhole = Math.round(position) const subPositionUnit = position - positionWhole positionInfo = { position, subPositionUnit, positionNonce: (curConfig.positionNonce + 1) % 256, minPosition: 0, maxPosition: Math.round( ((info.totalFrames - 1) * interfaceState.zoomTimelinePixelsPerFrame) / PIXELS_PER_POSITION, ), zoomTimelinePixelsPerFrame: interfaceState.zoomTimelinePixelsPerFrame, } } return { ...curConfig, ...positionInfo, detentPositions: nClosest, } }) } else if (currentMode === Mode.Frames) { setPlaybackState({ speed: 0, currentFrame: smartKnobState.currentPosition, }) // No config updates needed } else if (currentMode === Mode.Speed) { const normalizedWholeValue = smartKnobState.currentPosition const normalizedFractional = Math.sign(smartKnobState.subPositionUnit) * lerp(smartKnobState.subPositionUnit * Math.sign(smartKnobState.subPositionUnit), 0.1, 0.9, 0, 1) const normalized = normalizedWholeValue + normalizedFractional const speed = Math.sign(normalized) * Math.pow(2, Math.abs(normalized) - 1) const roundedSpeed = Math.trunc(speed * 10) / 10 setPlaybackState((cur) => { return { speed: roundedSpeed, currentFrame: cur.currentFrame, } }) // Update config with bounds depending on current frame setSmartKnobConfig((curConfig) => { return { ...curConfig, minPosition: playbackState.currentFrame === 0 ? 0 : -6, maxPosition: playbackState.currentFrame === info.totalFrames - 1 ? 0 : 6, } }) } else { exhaustiveCheck(currentMode) } }, [ detentPositions, nClosest, info.totalFrames, smartKnobState.config.text, smartKnobState.currentPosition, smartKnobState.subPositionUnit, smartKnobConfig.text, playbackState.currentFrame, playbackState.speed, interfaceState.zoomTimelinePixelsPerFrame, ]) // Change mode when pressed const receivedPressNonceRef = useRef(false) const previousPressNonceRef = useRef(0) useEffect(() => { if (previousPressNonceRef.current !== smartKnobState.pressNonce) { if (!receivedPressNonceRef.current) { // Ignore first nonce change receivedPressNonceRef.current = true } else { nextMode() } } previousPressNonceRef.current = smartKnobState.pressNonce }, [smartKnobState.pressNonce, nextMode]) const refreshInterval = 20 const updateFrame = useCallback(() => { const fps = info.frameRate * playbackState.speed setPlaybackState((cur) => { const newFrame = cur.currentFrame + (fps * refreshInterval) / 1000 const clampedNewFrame = Math.min(Math.max(newFrame, 0), info.totalFrames - 1) return { speed: cur.speed, currentFrame: clampedNewFrame, } }) }, [info.frameRate, playbackState.speed]) // Store the latest callback in a ref so the long-lived interval closure can invoke the latest version. // See https://overreacted.io/making-setinterval-declarative-with-react-hooks/ for more const savedCallback = useRef<() => void | null>() useEffect(() => { savedCallback.current = updateFrame }, [updateFrame]) const isPlaying = useMemo(() => { return playbackState.speed !== 0 }, [playbackState.speed]) useEffect(() => { if (smartKnobState.config.text === Mode.Speed && isPlaying) { const timer = setInterval(() => { if (savedCallback.current) { savedCallback.current() } }, refreshInterval) return () => clearInterval(timer) } }, [smartKnobState.config.text, isPlaying]) const connectToSerial = async () => { try { if (navigator.serial) { previousPressNonceRef.current = 0 receivedPressNonceRef.current = false const serialPort = await navigator.serial.requestPort({ filters: SmartKnobWebSerial.USB_DEVICE_FILTERS, }) serialPort.addEventListener('disconnect', () => { setSmartKnob(null) }) const smartKnob = new SmartKnobWebSerial(serialPort, (message) => { if (message.payload === 'smartknobState' && message.smartknobState !== null) { const state = PB.SmartKnobState.create(message.smartknobState) const stateObj = PB.SmartKnobState.toObject(state, { defaults: true, }) as NoUndefinedField setSmartKnobState(stateObj) } else if (message.payload === 'log' && message.log !== null) { console.log('LOG from smartknob', message.log?.msg) } }) setSmartKnob(smartKnob) await smartKnob.openAndLoop() } else { console.error('Web Serial API is not supported in this browser.') setSmartKnob(null) } } catch (error) { console.error('Error with serial port:', error) setSmartKnob(null) } } return ( <> Video Playback Control Demo {smartKnob !== null ? ( <> { if (value === null) { return } changeMode(value) }} aria-label="Mode" > {Object.entries(Mode).map((mode_entry) => ( {mode_entry[0]} ))} Frame {Math.trunc(playbackState.currentFrame)} / {info.totalFrames - 1}
Speed {playbackState.speed}
{ setInterfaceState((cur) => { const newZoom = Math.min( Math.max(cur.zoomTimelinePixelsPerFrame * factor, MIN_ZOOM), MAX_ZOOM, ) console.log(factor, newZoom) return { ...cur, zoomTimelinePixelsPerFrame: newZoom, } }) }} /> ) : navigator.serial ? ( ) : ( Sorry, Web Serial API isn't available in your browser. Try the latest version of Chrome. )}
{JSON.stringify(smartKnobConfig, undefined, 2)}
) } export type TimelineProps = { info: VideoInfo currentFrame: number zoomTimelinePixelsPerFrame: number adjustZoom: (factor: number) => void } export const Timeline: React.FC = ({info, currentFrame, zoomTimelinePixelsPerFrame, adjustZoom}) => { const gradients = [ 'linear-gradient( 135deg, #FDEB71 10%, #F8D800 100%)', 'linear-gradient( 135deg, #ABDCFF 10%, #0396FF 100%)', 'linear-gradient( 135deg, #FEB692 10%, #EA5455 100%)', 'linear-gradient( 135deg, #CE9FFC 10%, #7367F0 100%)', 'linear-gradient( 135deg, #90F7EC 10%, #32CCBC 100%)', ] const timelineRef = useRef(null) const cursorRef = useRef(null) useEffect(() => { const handleWheel = (event: HTMLElementEventMap['wheel']) => { const delta = event.deltaY if (delta) { event.preventDefault() adjustZoom(1 - delta / 500) } } timelineRef.current?.addEventListener('wheel', handleWheel) return () => { timelineRef.current?.removeEventListener('wheel', handleWheel) } }, []) useEffect(() => { cursorRef.current?.scrollIntoView() }, [currentFrame, zoomTimelinePixelsPerFrame]) return (
{[...info.boundaryFrames, info.totalFrames].map((f, i, a) => { const lengthFrames = f - (a[i - 1] ?? 0) const widthPixels = zoomTimelinePixelsPerFrame * lengthFrames return (
) })}
) }