Fix non-positional config changes and get dynamic detentPositions mostly working

firmware/src/motor_task.cpp

@@ -13,7 +13,7 @@
 // #### 
 // Hardware-specific motor calibration constants.
 // Run calibration once at startup, then update these constants with the calibration results.
-static const float ZERO_ELECTRICAL_OFFSET = 7.81;
+static const float ZERO_ELECTRICAL_OFFSET = 7.61;
 static const Direction FOC_DIRECTION = Direction::CW;
 static const int MOTOR_POLE_PAIRS = 7;
 // ####
@@ -96,8 +96,6 @@ void MotorTask::run() {
     uint32_t last_idle_start = 0;
     uint32_t last_publish = 0;
 
-    PB_SmartKnobConfig latest_config = config;
-
     while (1) {
         motor.loopFOC();
 
@@ -132,20 +130,22 @@ void MotorTask::run() {
                     }
 
                     // Change haptic input mode
-                    config = command.data.config;
-                    if (config.position == INT32_MIN) {
+                    PB_SmartKnobConfig newConfig = command.data.config;
+                    if (newConfig.position == INT32_MIN) {
                         // INT32_MIN indicates no change to position, so restore from latest_config
-                        config.position = latest_config.position;
+                        log("maintaining position");
+                        newConfig.position = config.position;
                     }
-                    if (config.position != latest_config.position
-                            || config.position_width_radians != latest_config.position_width_radians) {
+                    if (newConfig.position != config.position
+                            || newConfig.position_width_radians != config.position_width_radians) {
                         // Only adjust the detent center if the position or width has changed
+                        log("adjusting detent center");
                         current_detent_center = motor.shaft_angle;
                         #if SK_INVERT_ROTATION
                             current_detent_center = -motor.shaft_angle;
                         #endif
                     }
-                    latest_config = config;
+                    config = newConfig;
                     log("Got new config");
 
                     // Update derivative factor of torque controller based on detent width.

software/js/package-lock.json

@@ -9,11 +9,11 @@
       "version": "0.1.0",
       "license": "Apache-2.0",
       "workspaces": [
+        "packages/smartknobjs-proto",
+        "packages/smartknobjs",
         "packages/demo-backend",
         "packages/demo-frontend",
-        "packages/example",
-        "packages/smartknobjs-proto",
-        "packages/smartknobjs"
+        "packages/example"
       ],
       "devDependencies": {
         "concurrently": "^7.6.0",
@@ -21796,7 +21796,7 @@
         "@testing-library/react": "^13.4.0",
         "@testing-library/user-event": "^13.5.0",
         "@types/jest": "^27.5.2",
-        "@types/lodash": "*",
+        "@types/lodash": "^4.14.191",
         "@types/node": "^16.18.4",
         "@types/react": "^18.0.25",
         "@types/react-dom": "^18.0.9",

software/js/packages/demo-frontend/src/App.tsx

@@ -1,4 +1,4 @@
-import React, {useEffect, useState} from 'react'
+import React, {useEffect, useMemo, useRef, useState} from 'react'
 import './App.css'
 import io from 'socket.io-client'
 import Typography from '@mui/material/Typography'
@@ -7,13 +7,16 @@ import ToggleButton from '@mui/material/ToggleButton'
 import ToggleButtonGroup from '@mui/material/ToggleButtonGroup'
 import {PB} from 'smartknobjs-proto'
 import {VideoInfo} from './types'
-import {Card, CardActions, CardContent} from '@mui/material'
-import {exhaustiveCheck, INT32_MIN, lerp, NoUndefinedField} from './util'
+import {Card, CardContent} from '@mui/material'
+import {exhaustiveCheck, findNClosest, INT32_MIN, lerp, NoUndefinedField} from './util'
+import {groupBy, parseInt} from 'lodash'
 
 const socket = io()
 
-// const MIN_ZOOM = 0.01
-// const MAX_ZOOM = 40
+const MIN_ZOOM = 0.01
+const MAX_ZOOM = 60
+
+const PIXELS_PER_POSITION = 10
 
 enum Mode {
     Scroll = 'Scroll',
@@ -33,8 +36,11 @@ export type AppProps = {
 }
 export const App: React.FC<AppProps> = ({info}) => {
     const [isConnected, setIsConnected] = useState(socket.connected)
-    const [state, setState] = useState<string>('')
-
+    const [state, setState] = useState<NoUndefinedField<PB.ISmartKnobState>>(
+        PB.SmartKnobState.toObject(PB.SmartKnobState.create({config: PB.SmartKnobConfig.create()}), {
+            defaults: true,
+        }) as NoUndefinedField<PB.ISmartKnobState>,
+    )
     const [derivedState, setDerivedState] = useState<State>({
         mode: Mode.Scroll,
         playbackSpeed: 0,
@@ -42,20 +48,91 @@ export const App: React.FC<AppProps> = ({info}) => {
         zoomTimelinePixelsPerFrame: 0.1,
     })
 
+    useMemo(() => {
+        setDerivedState((cur) => {
+            const modeText = state.config.text
+            if (modeText === Mode.Scroll) {
+                const rawFrame = Math.trunc(
+                    ((state.currentPosition + state.subPositionUnit) * PIXELS_PER_POSITION) /
+                        cur.zoomTimelinePixelsPerFrame,
+                )
+                return {
+                    mode: Mode.Scroll,
+                    playbackSpeed: 0,
+                    currentFrame: Math.min(Math.max(rawFrame, 0), info.totalFrames - 1),
+                    zoomTimelinePixelsPerFrame: cur.zoomTimelinePixelsPerFrame,
+                }
+            } else if (modeText === Mode.Frames) {
+                return {
+                    mode: Mode.Frames,
+                    playbackSpeed: 0,
+                    currentFrame: state.currentPosition ?? 0,
+                    zoomTimelinePixelsPerFrame: cur.zoomTimelinePixelsPerFrame,
+                }
+            } else if (modeText === Mode.Speed) {
+                const normalizedWholeValue = state.currentPosition
+                const normalizedFractional =
+                    Math.sign(state.subPositionUnit) *
+                    lerp(state.subPositionUnit * Math.sign(state.subPositionUnit), 0.1, 0.9, 0, 1)
+                const normalized = normalizedWholeValue + normalizedFractional
+                const speed = Math.sign(normalized) * Math.pow(2, Math.abs(normalized) - 1)
+                return {
+                    mode: Mode.Speed,
+                    playbackSpeed: speed,
+                    currentFrame: cur.currentFrame,
+                    zoomTimelinePixelsPerFrame: cur.zoomTimelinePixelsPerFrame,
+                }
+            }
+            return cur
+        })
+    }, [state.config.text, state.currentPosition, state.subPositionUnit])
+
+    const totalPositions = Math.ceil((info.totalFrames * derivedState.zoomTimelinePixelsPerFrame) / PIXELS_PER_POSITION)
+    const detentPositions = useMemo(() => {
+        // Always include the first and last positions at detents
+        const positionsToFrames = groupBy([0, ...info.boundaryFrames, info.totalFrames - 1], (frame) =>
+            Math.round((frame * derivedState.zoomTimelinePixelsPerFrame) / PIXELS_PER_POSITION),
+        )
+        console.log(JSON.stringify(positionsToFrames))
+        return positionsToFrames
+    }, [info.boundaryFrames, totalPositions, derivedState.zoomTimelinePixelsPerFrame])
+
+    // Continuous config updates for scrolling, to update detent positions
+    useMemo(() => {
+        if (derivedState.mode === Mode.Scroll) {
+            const config = PB.SmartKnobConfig.create({
+                position: INT32_MIN,
+                minPosition: 0,
+                maxPosition: totalPositions - 1,
+                positionWidthRadians: (8 * Math.PI) / 180,
+                detentStrengthUnit: 2.5,
+                endstopStrengthUnit: 1,
+                snapPoint: 0.7,
+                text: Mode.Scroll,
+                detentPositions: findNClosest(Object.keys(detentPositions).map(parseInt), state.currentPosition, 5),
+                snapPointBias: 0,
+            })
+            socket.emit('set_config', config)
+        }
+    }, [derivedState.mode, derivedState.zoomTimelinePixelsPerFrame, detentPositions, state.currentPosition])
+
+    // For one-off config pushes, e.g. mode changes
     const pushConfig = (state: State) => {
         let config: PB.SmartKnobConfig
         if (state.mode === Mode.Scroll) {
-            // TODO
+            const position = Math.trunc((state.currentFrame * state.zoomTimelinePixelsPerFrame) / PIXELS_PER_POSITION)
             config = PB.SmartKnobConfig.create({
-                position: 0,
+                position,
                 minPosition: 0,
-                maxPosition: 31,
-                positionWidthRadians: (7 * Math.PI) / 180,
+                maxPosition: Math.trunc(
+                    ((info.totalFrames - 1) * state.zoomTimelinePixelsPerFrame) / PIXELS_PER_POSITION,
+                ),
+                positionWidthRadians: (8 * Math.PI) / 180,
                 detentStrengthUnit: 2.5,
                 endstopStrengthUnit: 1,
                 snapPoint: 0.7,
                 text: Mode.Scroll,
-                detentPositions: [0, 10, 21, 22],
+                detentPositions: findNClosest(Object.keys(detentPositions).map(parseInt), position, 5),
                 snapPointBias: 0,
             })
         } else if (state.mode === Mode.Frames) {
@@ -63,7 +140,7 @@ export const App: React.FC<AppProps> = ({info}) => {
                 position: state.currentFrame,
                 minPosition: 0,
                 maxPosition: info.totalFrames - 1,
-                positionWidthRadians: (1 * Math.PI) / 180,
+                positionWidthRadians: (1.5 * Math.PI) / 180,
                 detentStrengthUnit: 1,
                 endstopStrengthUnit: 1,
                 snapPoint: 1.1,
@@ -100,6 +177,7 @@ export const App: React.FC<AppProps> = ({info}) => {
         })
     }
 
+    // Timer for speed-based playback
     useEffect(() => {
         const refreshInterval = 20
         const fps = info.frameRate * derivedState.playbackSpeed
@@ -113,13 +191,15 @@ export const App: React.FC<AppProps> = ({info}) => {
 
                     if (newFrame < 0 || newFrame >= info.totalFrames) {
                         const clampedNewFrame = Math.min(Math.max(newFrame, 0), info.totalFrames - 1)
-                        // TODO: figure out how to avoid sending this config repeatedly
-                        pushConfig({
-                            mode: Mode.Speed,
-                            playbackSpeed: 0,
-                            currentFrame: Math.trunc(clampedNewFrame),
-                            zoomTimelinePixelsPerFrame: derivedState.zoomTimelinePixelsPerFrame,
-                        })
+                        if (oldFrame !== clampedNewFrame) {
+                            // If we've hit a boundary, push a config to set the bounds
+                            pushConfig({
+                                mode: Mode.Speed,
+                                playbackSpeed: 0,
+                                currentFrame: Math.trunc(clampedNewFrame),
+                                zoomTimelinePixelsPerFrame: derivedState.zoomTimelinePixelsPerFrame,
+                            })
+                        }
                         return clampedNewFrame
                     } else {
                         if (
@@ -142,6 +222,7 @@ export const App: React.FC<AppProps> = ({info}) => {
         }
     }, [derivedState.mode, derivedState.playbackSpeed, info.totalFrames, info.frameRate])
 
+    // Socket.io subscription
     useEffect(() => {
         socket.on('connect', () => {
             setIsConnected(true)
@@ -157,39 +238,7 @@ export const App: React.FC<AppProps> = ({info}) => {
             const stateObj = PB.SmartKnobState.toObject(state, {
                 defaults: true,
             }) as NoUndefinedField<PB.ISmartKnobState>
-            setState(JSON.stringify(stateObj, undefined, 4))
-            setDerivedState((cur) => {
-                const modeText = stateObj.config.text
-                if (modeText === Mode.Scroll) {
-                    return {
-                        mode: Mode.Scroll,
-                        playbackSpeed: 0,
-                        currentFrame: cur.currentFrame, // TODO - map from state and zoom
-                        zoomTimelinePixelsPerFrame: cur.zoomTimelinePixelsPerFrame,
-                    }
-                } else if (modeText === Mode.Frames) {
-                    return {
-                        mode: Mode.Frames,
-                        playbackSpeed: 0,
-                        currentFrame: stateObj.currentPosition ?? 0,
-                        zoomTimelinePixelsPerFrame: cur.zoomTimelinePixelsPerFrame,
-                    }
-                } else if (modeText === Mode.Speed) {
-                    const normalizedWholeValue = stateObj.currentPosition
-                    const normalizedFractional =
-                        Math.sign(stateObj.subPositionUnit) *
-                        lerp(stateObj.subPositionUnit * Math.sign(stateObj.subPositionUnit), 0.1, 0.9, 0, 1)
-                    const normalized = normalizedWholeValue + normalizedFractional
-                    const speed = Math.sign(normalized) * Math.pow(2, Math.abs(normalized) - 1)
-                    return {
-                        mode: Mode.Speed,
-                        playbackSpeed: speed,
-                        currentFrame: cur.currentFrame,
-                        zoomTimelinePixelsPerFrame: cur.zoomTimelinePixelsPerFrame,
-                    }
-                }
-                return cur
-            })
+            setState(stateObj)
         })
         return () => {
             socket.off('connect')
@@ -198,64 +247,162 @@ export const App: React.FC<AppProps> = ({info}) => {
         }
     }, [])
     return (
-        <Container component="main" maxWidth="sm">
-            <Card>
-                <CardContent>
-                    <Typography component="h1" variant="h5">
-                        Video Playback Control Demo
-                    </Typography>
-                    <ToggleButtonGroup
-                        color="primary"
-                        value={derivedState.mode}
-                        exclusive
-                        onChange={(e, value: Mode | null) => {
-                            if (value === null) {
-                                return
-                            }
-                            pushConfig({
-                                ...derivedState,
-                                mode: value,
-                            })
-                        }}
-                        aria-label="Mode"
-                    >
-                        {Object.keys(Mode).map((mode) => (
-                            <ToggleButton value={mode} key={mode}>
-                                {mode}
-                            </ToggleButton>
-                        ))}
-                    </ToggleButtonGroup>
-                    <Typography component="h1" variant="h5">
-                        Frame {Math.trunc(derivedState.currentFrame)} / {info.totalFrames - 1}
-                        <br />
-                        Speed {Math.trunc(derivedState.playbackSpeed * 10) / 10}
-                    </Typography>
-                </CardContent>
-                <CardActions>
-                    {/* {derivedState.mode === Mode.Speed && (
-                        <Button
-                            size="small"
-                            onClick={() => {
-                                if (derivedState.playbackSpeed !== 0) {
-                                    setPlaybackSpeed(0)
-                                } else {
-                                    setPlaybackSpeed(1)
+        <>
+            <Container component="main" maxWidth="md">
+                <Card>
+                    <CardContent>
+                        <Typography component="h1" variant="h5">
+                            Video Playback Control Demo
+                        </Typography>
+                        {isConnected || (
+                            <Typography component="h6" variant="h6">
+                                [Not connected]
+                            </Typography>
+                        )}
+                        <ToggleButtonGroup
+                            color="primary"
+                            value={derivedState.mode}
+                            exclusive
+                            onChange={(e, value: Mode | null) => {
+                                if (value === null) {
+                                    return
                                 }
-                                pushCurrentConfig()
+                                pushConfig({
+                                    ...derivedState,
+                                    mode: value,
+                                })
                             }}
+                            aria-label="Mode"
                         >
-                            Play
-                        </Button>
-                    )} */}
-                </CardActions>
-            </Card>
-            <Card>
-                <CardContent>
-                    <div>{isConnected ? <code>{state}</code> : <p>Not connected</p>}</div>
-                </CardContent>
-            </Card>
-        </Container>
+                            {Object.keys(Mode).map((mode) => (
+                                <ToggleButton value={mode} key={mode}>
+                                    {mode}
+                                </ToggleButton>
+                            ))}
+                        </ToggleButtonGroup>
+                        <Typography>
+                            Frame {Math.trunc(derivedState.currentFrame)} / {info.totalFrames - 1}
+                            <br />
+                            Speed {Math.trunc(derivedState.playbackSpeed * 10) / 10}
+                        </Typography>
+                    </CardContent>
+                </Card>
+                <Timeline
+                    info={info}
+                    currentFrame={derivedState.currentFrame}
+                    zoomTimelinePixelsPerFrame={derivedState.zoomTimelinePixelsPerFrame}
+                    adjustZoom={(factor) => {
+                        setDerivedState((cur) => {
+                            const newZoom = Math.min(
+                                Math.max(cur.zoomTimelinePixelsPerFrame * factor, MIN_ZOOM),
+                                MAX_ZOOM,
+                            )
+                            console.log(factor, newZoom)
+                            return {
+                                ...cur,
+                                zoomTimelinePixelsPerFrame: newZoom,
+                            }
+                        })
+                    }}
+                />
+                <Card>
+                    <CardContent>
+                        <div>{JSON.stringify(detentPositions)}</div>
+                    </CardContent>
+                </Card>
+            </Container>
+        </>
     )
 }
 
-export default App
+export type TimelineProps = {
+    info: VideoInfo
+    currentFrame: number
+    zoomTimelinePixelsPerFrame: number
+    adjustZoom: (factor: number) => void
+}
+export const Timeline: React.FC<TimelineProps> = ({info, currentFrame, zoomTimelinePixelsPerFrame, adjustZoom}) => {
+    const gradients = [
+        'linear-gradient( 135deg, #FDEB71 10%, #F8D800 100%)',
+        'linear-gradient( 135deg, #ABDCFF 10%, #0396FF 100%)',
+        'linear-gradient( 135deg, #FEB692 10%, #EA5455 100%)',
+        'linear-gradient( 135deg, #CE9FFC 10%, #7367F0 100%)',
+        'linear-gradient( 135deg, #90F7EC 10%, #32CCBC 100%)',
+    ]
+
+    const timelineRef = useRef<HTMLDivElement>(null)
+    const cursorRef = useRef<HTMLDivElement>(null)
+
+    useEffect(() => {
+        const handleWheel = (event: HTMLElementEventMap['wheel']) => {
+            const delta = event.deltaY
+            if (delta) {
+                event.preventDefault()
+                adjustZoom(1 - delta / 500)
+            }
+        }
+
+        timelineRef.current?.addEventListener('wheel', handleWheel)
+        return () => {
+            timelineRef.current?.removeEventListener('wheel', handleWheel)
+        }
+    }, [])
+
+    useEffect(() => {
+        cursorRef.current?.scrollIntoView()
+    }, [currentFrame, zoomTimelinePixelsPerFrame])
+    return (
+        <div
+            className="timeline-container"
+            ref={timelineRef}
+            style={{
+                width: '100%',
+                margin: '10px auto',
+                overflowX: 'scroll',
+            }}
+        >
+            <div
+                className="timeline"
+                style={{
+                    position: 'relative',
+                    display: 'inline-block',
+                    height: '80px',
+                    width: `${zoomTimelinePixelsPerFrame * info.totalFrames}px`,
+                    backgroundColor: '#dde',
+                }}
+            >
+                {[...info.boundaryFrames, info.totalFrames].map((f, i, a) => {
+                    const lengthFrames = f - (a[i - 1] ?? 0)
+                    const widthPixels = zoomTimelinePixelsPerFrame * lengthFrames
+                    return (
+                        <div
+                            key={`clip-${f}`}
+                            className="video-clip"
+                            style={{
+                                position: 'relative',
+                                display: 'inline-block',
+                                top: '10px',
+                                height: '60px',
+                                width: `${widthPixels}px`,
+                                backgroundImage: gradients[i % gradients.length],
+                            }}
+                        ></div>
+                    )
+                })}
+                <div
+                    className="playback-cursor"
+                    ref={cursorRef}
+                    style={{
+                        position: 'absolute',
+                        display: 'inline-block',
+                        left: `${zoomTimelinePixelsPerFrame * Math.trunc(currentFrame)}px`,
+                        width: `${Math.max(zoomTimelinePixelsPerFrame, 1)}px`,
+                        height: '100%',
+                        backgroundColor: 'rgba(255, 0, 0, 0.4)',
+                        borderLeft: '1px solid red',
+                    }}
+                ></div>
+            </div>
+        </div>
+    )
+}

software/js/packages/demo-frontend/src/index.tsx

@@ -1,7 +1,7 @@
 import React from 'react'
 import ReactDOM from 'react-dom/client'
 import './index.css'
-import App from './App'
+import {App} from './App'
 import reportWebVitals from './reportWebVitals'
 import '@fontsource/roboto/300.css'
 import '@fontsource/roboto/400.css'
@@ -18,7 +18,7 @@ const root = ReactDOM.createRoot(document.getElementById('root') as HTMLElement)
 const info: VideoInfo = {
     totalFrames: 30 * 60 * 5,
     frameRate: 30,
-    boundaryFrames: [1234, 1290, 3000],
+    boundaryFrames: [312, 400, 1234, 1290, 3000, 4000],
 }
 root.render(
     <React.StrictMode>

software/js/packages/demo-frontend/src/util.ts

@@ -19,3 +19,13 @@ export type NoUndefinedField<T> = {
 }
 
 export const INT32_MIN = -2147483648
+
+export function findNClosest(numbers: number[], target: number, n: number): number[] {
+    // First, we sort the numbers in ascending order based on their absolute difference
+    // from the target number. This means that the numbers that are closest to the target
+    // will come first in the sorted array.
+    const sortedNumbers = numbers.sort((a, b) => Math.abs(a - target) - Math.abs(b - target))
+
+    // Next, we return the first N numbers from the sorted array as the N closest numbers.
+    return sortedNumbers.slice(0, n)
+}