diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 3f9861e2..0e06671b 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -20,6 +20,15 @@ This assumes you have the following technologies installed and on your path: I recommend using something like [nvm](https://github.com/nvm-sh/nvm#installing-and-updating) for installing and managing versions of node but any method will work. +#### Quick setup (recommended) + +A `setup.sh` script is provided at the root of the repository that automates the installation of all required tools and dependencies. It will install nvm, Node.js, Yarn, and all project packages: + +```bash +$ bash setup.sh +``` + +#### Manual setup 1. Fork the repo (Once we see any semi-serious input from a developer we will grant write permissions to our central repository. You can also request this earlier if you wish.) 2. Install dependencies @@ -33,6 +42,39 @@ SERVER= yarn start Then you can go to http://localhost:8000 (use `PORT` env variable to run on a different port) in your browser. 4. After you've made your changes, run `yarn lint`, `yarn test` and then open a PR. +### Running in the background + +A `run_server` script is provided at the root of the repository to start the development server as a background process. It requires the URL of your running FlexGet instance as its only argument: + +```bash +$ ./run_server http://: +``` + +For example: + +```bash +$ ./run_server http://192.168.1.228:5050 +``` + +Once started, the WebUI will be available at http://localhost:8000. Output from the server is written to `.webui.log` in the project root. + +If the server is already running, the script will report its PID and exit without starting a second instance. + +#### Stopping the server + +Add the `stop-webui` shell alias to your environment by sourcing your `~/.bashrc`: + +```bash +# alias stop-webui='pkill -f "babel-node.*server.js" 2>/dev/null && rm -f ~/personal/webui/.webui.pid && echo "WebUI stopped" || echo "WebUI is not running"' +$ source ~/.bashrc +``` + +Then stop the server at any time with: + +```bash +$ stop-webui +``` + ### Notes * We are in the process of moving from javascript to typescript and if you are making substantial changes to a javascript file, please convert it to typescript if you feel comfortable doing so. All new files should be written in typescript. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..458e8772 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,55 @@ +# Project-specific guidance + +## Stack +React 16 + TypeScript 4, Material UI v4, Emotion CSS, Formik, Webpack 4, Yarn 1.x. +Run tests: `yarn test --no-coverage` (requires Node 16 via nvm). + +## Plugin registration +New plugins live in `src/plugins//`. Register in `src/Root.tsx` by importing +and calling the default export from `src/plugins//index.ts`, which calls +`registerPlugin(path, { component, displayName, icon })`. +To hide a plugin from the sidebar without removing its route, pass `hidden: true` +to `registerPlugin`. The `SideNav` filters it out; `Routes` still registers it. + +## API hooks +- `useFlexgetAPI(url, method)` — REST calls; URL is fixed at hook creation time. +- `useFlexgetStream(url, method)` — oboe streaming; returns `[{ stream, readyState }, { connect, disconnect }]`. + Attach `.node()`, `.done()`, `.fail()` handlers in a `useEffect([stream])` — the hook itself only + handles `.start()` and `.fail()` for `ReadyState`. There is NO built-in `.done()` handler, so + `readyState` never returns to `Closed` on a successful stream; attach `.done()` directly on the + stream object to detect completion. +- Request bodies are auto-converted to snake_case; responses are auto-camelized. +- `useFlexgetAPI` URL is fixed per render. For DELETE/PUT calls where the path param + changes at submit time (e.g. task name edited by user), store the value in `useState` + and pass it to the hook — a re-render updates the request fn. When that request fires + after an async operation (e.g. a stream `.done()`), hold the callback in a `useRef` + so the stream effect doesn't list the callback as a dep and won't re-attach handlers + on re-render: `const ref = useRef(fn); useEffect(() => { ref.current = fn; }, [fn]);` + +## MUI + Emotion css prop conflict +When extending `React.HTMLAttributes` for a component that renders +inside MUI/Emotion, use `Omit, 'css'>`. +Emotion globally augments `HTMLAttributes` with its own `css` type, which conflicts +with MUI Box's `css` prop, causing a TS error at component definition time. + +## Testing +- MUI v4 `TextField` without an explicit `id` prop doesn't wire `htmlFor` in JSDOM. + Use `container.querySelector('[name="fieldName"]')` instead of `getByLabelText`. +- `@testing-library/react` v9 has no `name` option on `getByRole`. + Use `getByText('Label').closest('button')` for buttons. +- `act` is not exported from `@testing-library/react` v9; import from `react-dom/test-utils`. +- To mock `useFlexgetStream` in tests, use `jest.spyOn(coreApi, 'useFlexgetStream')` — + ts-jest compiles to CommonJS so named-import spying works. +- Async tests that involve navigation → API fetch → Formik reinitialize need extended + timeouts; set `jest.setTimeout(15000)` in `beforeAll`. +- MUI v4 `Select` doesn't wire `[data-value]` reliably in JSDOM. Open with + `fireEvent.mouseDown(selectEl)`, then find options via + `document.querySelectorAll('[role="option"]')` (they render into a portal) and + match by `el.textContent?.trim()`. +- When multiple `Select` components are on the page, `.MuiSelect-root` indices follow + JSX render order, not visual position. Confirm the index of the target Select before + using it in a test. +- When `fetchMock` returns a non-ok status the error object carries a `message` string + (the HTTP status text), so a `?? 'Unknown error'` fallback is never reached that way. + To test the unknown-error path, mock the hook directly: + `jest.spyOn(hooks, 'useCreate...').mockReturnValue([..., jest.fn().mockResolvedValue({ ok: false, error: undefined })])` diff --git a/run_server.sh b/run_server.sh new file mode 100755 index 00000000..c973287e --- /dev/null +++ b/run_server.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PID_FILE="$SCRIPT_DIR/.webui.pid" +LOG_FILE="$SCRIPT_DIR/.webui.log" + +if [ -z "${1:-}" ]; then + echo "Usage: run_server " + echo " e.g. run_server http://192.168.1.228:5050" + exit 1 +fi + +FLEXGET_SERVER="$1" + +if [ -f "$PID_FILE" ] && kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then + echo "WebUI is already running (PID $(cat "$PID_FILE"))" + echo " Log: $LOG_FILE" + echo " Stop: stop-webui" + exit 0 +fi + +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +[ -s "$NVM_DIR/nvm.sh" ] && source "$NVM_DIR/nvm.sh" + +cd "$SCRIPT_DIR" + +SERVER="$FLEXGET_SERVER" yarn start >"$LOG_FILE" 2>&1 & +echo $! >"$PID_FILE" + +echo "WebUI started (PID $!)" +echo " URL: http://localhost:8000" +echo " API: $FLEXGET_SERVER" +echo " Log: $LOG_FILE" +echo " Stop: stop-webui" diff --git a/setup.sh b/setup.sh new file mode 100755 index 00000000..e73e5660 --- /dev/null +++ b/setup.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Minimum required versions +NODE_MAJOR=16 +YARN_VERSION="1.22.22" + +log() { echo "[setup] $*"; } +err() { echo "[setup] ERROR: $*" >&2; exit 1; } + +# Install nvm if not present +if ! command -v nvm &>/dev/null && [ ! -f "$HOME/.nvm/nvm.sh" ]; then + log "Installing nvm..." + curl -fsSL https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash +fi + +# Load nvm +export NVM_DIR="${NVM_DIR:-$HOME/.nvm}" +# shellcheck source=/dev/null +[ -s "$NVM_DIR/nvm.sh" ] && source "$NVM_DIR/nvm.sh" + +if ! command -v nvm &>/dev/null; then + err "nvm not found after install — open a new shell and re-run this script" +fi + +# Install and use the required Node version +log "Installing Node.js $NODE_MAJOR (LTS)..." +nvm install "$NODE_MAJOR" +nvm use "$NODE_MAJOR" +log "Node $(node --version)" + +# Install Yarn classic (v1) +if ! command -v yarn &>/dev/null || [[ "$(yarn --version)" != 1.* ]]; then + log "Installing Yarn $YARN_VERSION..." + npm install -g "yarn@$YARN_VERSION" +fi +log "Yarn $(yarn --version)" + +# Install project dependencies +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +log "Installing project dependencies..." +yarn install --frozen-lockfile + +log "" +log "Setup complete. Available commands:" +log " yarn start — start the dev server" +log " yarn build — production build" +log " yarn test — run tests" +log " yarn lint — lint TypeScript/JavaScript" diff --git a/src/Root.tsx b/src/Root.tsx index a72a5bb4..4ba5f081 100644 --- a/src/Root.tsx +++ b/src/Root.tsx @@ -15,6 +15,7 @@ import registerConfig from 'plugins/config'; import registerPendingList from 'plugins/lists/pending'; import registerMovieList from 'plugins/lists/movies'; import registerEntryList from 'plugins/lists/entry'; +import registerBackfill from 'plugins/backfill'; import registerOperations from 'core/operations'; import { AuthContainer } from 'core/auth/hooks'; import { TaskContainer } from 'plugins/tasks/hooks'; @@ -33,6 +34,7 @@ registerSeries(); registerPendingList(); registerEntryList(); registerMovieList(); +registerBackfill(); const globals = css` html { diff --git a/src/common/inputs/formik/TextField.tsx b/src/common/inputs/formik/TextField.tsx index 04c36e93..1822584f 100644 --- a/src/common/inputs/formik/TextField.tsx +++ b/src/common/inputs/formik/TextField.tsx @@ -6,10 +6,17 @@ export type Props = TextFieldProps & { name: string; }; -const TextField: FC = ({ name, ...props }) => { +const TextField: FC = ({ name, helperText, ...props }) => { const [field, { touched, error }] = useField(name); - return ; + return ( + + ); }; export default TextField; diff --git a/src/core/layout/SideNav/index.tsx b/src/core/layout/SideNav/index.tsx index 45e7037f..70034ef2 100644 --- a/src/core/layout/SideNav/index.tsx +++ b/src/core/layout/SideNav/index.tsx @@ -92,7 +92,7 @@ const SideNav: FC = ({ sidebarOpen = false, onClose, className }) => { width: inherit; `} > - {routes.map(route => ( + {routes.filter(route => !route.hidden).map(route => ( ))} diff --git a/src/core/layout/__snapshots__/Layout.spec.tsx.snap b/src/core/layout/__snapshots__/Layout.spec.tsx.snap index b217a194..1e3caa8b 100644 --- a/src/core/layout/__snapshots__/Layout.spec.tsx.snap +++ b/src/core/layout/__snapshots__/Layout.spec.tsx.snap @@ -395,10 +395,10 @@ exports[`common/layout renders correctly 1`] = ` Array [ Object { "map": undefined, - "name": "1a7v7et", + "name": "10zn699", "next": undefined, "styles": " - overflow-y: auto; + overflow-y: scroll; padding: 1.6rem; height: 100%; diff --git a/src/core/layout/styles.ts b/src/core/layout/styles.ts index daa48790..428729bf 100644 --- a/src/core/layout/styles.ts +++ b/src/core/layout/styles.ts @@ -42,7 +42,7 @@ export const leavingTransition = (theme: Theme) => css` `; export const content = (theme: Theme) => css` - overflow-y: auto; + overflow-y: scroll; padding: ${theme.typography.pxToRem(theme.spacing(2))}; height: 100%; diff --git a/src/core/plugins/types.ts b/src/core/plugins/types.ts index 7887cefc..28f6ca18 100644 --- a/src/core/plugins/types.ts +++ b/src/core/plugins/types.ts @@ -5,6 +5,7 @@ export interface Plugin { displayName: string; icon: ComponentType; cardComponent?: ComponentType; + hidden?: boolean; } export type PluginMap = Record; export type PluginUpdateHandler = (e: CustomEvent) => void; diff --git a/src/core/routes/hooks.ts b/src/core/routes/hooks.ts index 173d93c8..3c992793 100644 --- a/src/core/routes/hooks.ts +++ b/src/core/routes/hooks.ts @@ -7,7 +7,7 @@ export const useGetRoutes = () => { const { pluginMap } = useContainer(PluginContainer); const routes: Route[] = useMemo( () => - Object.entries(pluginMap).flatMap(([path, { component, displayName, icon }]) => + Object.entries(pluginMap).flatMap(([path, { component, displayName, icon, hidden }]) => component ? [ { @@ -15,6 +15,7 @@ export const useGetRoutes = () => { component, Icon: icon, name: displayName, + hidden, }, ] : [], diff --git a/src/core/routes/types.ts b/src/core/routes/types.ts index 87a32929..2d071fae 100644 --- a/src/core/routes/types.ts +++ b/src/core/routes/types.ts @@ -5,4 +5,5 @@ export interface Route { name: string; Icon: ComponentType; path: string; + hidden?: boolean; } diff --git a/src/plugins/backfill/Backfill.spec.tsx b/src/plugins/backfill/Backfill.spec.tsx new file mode 100644 index 00000000..2f5f42f3 --- /dev/null +++ b/src/plugins/backfill/Backfill.spec.tsx @@ -0,0 +1,705 @@ +import React, { FC, useEffect } from 'react'; +import { act } from 'react-dom/test-utils'; +import { cleanup, fireEvent, wait } from '@testing-library/react'; +import { useHistory, Route, Switch } from 'react-router'; +import fetchMock from 'fetch-mock'; +import YAML from 'yaml'; +import { renderWithWrapper } from 'utils/tests'; +import AppBar from 'core/layout/AppBar'; +import { TaskContainer } from 'plugins/tasks/hooks'; +import * as coreApi from 'core/api'; +import * as backfillHooks from './hooks'; +import Backfill from './Backfill'; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +interface Props { + path: string; +} + +const TestBackfill: FC = ({ path }) => { + const { push } = useHistory(); + useEffect(() => { push(path); }, [path, push]); + return ( + + + + + + + ); +}; + +const getField = (container: HTMLElement, name: string) => + container.querySelector(`[name="${name}"]`) as HTMLInputElement | HTMLTextAreaElement | null; + +const taskConfig = { + config: { rss: { url: 'http://source.example.com/rss?q=base&cat=1' }, series: ['Breaking Bad', 'The Wire'] }, + name: 'test-task', +}; + +const waitForConfig = (container: HTMLElement) => + wait( + () => { + const el = getField(container, 'taskConfig'); + expect(el).not.toBeNull(); + expect(el!.value).toBe(YAML.stringify(taskConfig)); + }, + { timeout: 8000 }, + ); + +// RssBackfillUrlField (Query Param) is rendered before TaskSeriesField in the JSX, +// so the Task Series Select is always at MuiSelect-root index 1. +const QUERY_PARAM_SELECT_INDEX = 0; +const TASK_SERIES_SELECT_INDEX = 1; + +// Open a MUI Select by DOM index and click the option matching `text`. +// MUI renders the listbox into a portal so we query document directly. +const pickSelectOption = async (container: HTMLElement, index: number, text: string) => { + const selectDiv = container.querySelectorAll('.MuiSelect-root')[index] as HTMLElement; + fireEvent.mouseDown(selectDiv); + await wait(() => { + const options = Array.from(document.querySelectorAll('[role="option"]')); + const option = options.find(el => el.textContent?.trim() === text); + expect(option).not.toBeNull(); + fireEvent.click(option!); + }); +}; + +const pickSeriesOption = (container: HTMLElement, text: string) => + pickSelectOption(container, TASK_SERIES_SELECT_INDEX, text); + +// --------------------------------------------------------------------------- +// Setup +// --------------------------------------------------------------------------- + +describe('plugins/backfill/Backfill', () => { + beforeAll(() => jest.setTimeout(15000)); + afterAll(() => jest.setTimeout(5000)); + + let capturedDoneCallback: (() => void) | undefined; + let capturedFailCallback: (() => void) | undefined; + let capturedNodeHandlers: Record void>; + let mockConnect: jest.Mock; + let mockStream: any; + + beforeEach(() => { + capturedDoneCallback = undefined; + capturedFailCallback = undefined; + capturedNodeHandlers = {}; + mockConnect = jest.fn(); + + mockStream = { + node: jest.fn().mockImplementation((path: string, cb: (e: any) => void) => { + capturedNodeHandlers[path] = cb; + return mockStream; + }), + done: jest.fn().mockImplementation((cb: () => void) => { + capturedDoneCallback = cb; + return mockStream; + }), + fail: jest.fn().mockImplementation((cb: () => void) => { + capturedFailCallback = cb; + return mockStream; + }), + }; + + jest.spyOn(coreApi, 'useFlexgetStream').mockReturnValue([ + { stream: mockStream as any, readyState: coreApi.ReadyState.Closed }, + { connect: mockConnect, disconnect: jest.fn() }, + ]); + + fetchMock + .get('/api/tasks', []) + .get('/api/tasks/test-task', taskConfig) + .post('/api/tasks', 200) + .delete('/api/tasks/test-task-backfill', 204) + .catch(); + }); + + afterEach(() => { + cleanup(); + fetchMock.reset(); + jest.restoreAllMocks(); + }); + + // ------------------------------------------------------------------------- + // Initial state + // ------------------------------------------------------------------------- + + describe('initial state', () => { + it('derives taskName from the task query param', async () => { + const { container } = renderWithWrapper(); + await waitForConfig(container); + expect(getField(container, 'taskName')?.value).toBe('test-task-backfill'); + }); + + it('initialises rssBackfillUrl from the loaded task config', async () => { + const { container } = renderWithWrapper(); + await waitForConfig(container); + expect(getField(container, 'rssBackfillUrl')?.value).toBe( + 'http://source.example.com/rss?q=base&cat=1', + ); + }); + + it('shows "No task selected" in taskConfig when no task param', async () => { + const { container } = renderWithWrapper(); + await wait(() => expect(getField(container, 'taskConfig')).not.toBeNull()); + expect(getField(container, 'taskConfig')?.value).toBe('No task selected'); + }); + + it('auto-update checkbox is checked after config loads', async () => { + const { container } = renderWithWrapper(); + await waitForConfig(container); + const checkbox = container.querySelector('input[type="checkbox"]') as HTMLInputElement; + expect(checkbox.checked).toBe(true); + }); + }); + + // ------------------------------------------------------------------------- + // Task Series dropdown + // ------------------------------------------------------------------------- + + describe('Task Series dropdown', () => { + it('appears when the config contains a series key', async () => { + const { queryByText, container } = renderWithWrapper( + , + ); + await waitForConfig(container); + expect(queryByText('Task Series')).toBeInTheDocument(); + }); + + it('shows a disabled Select with placeholder text when the config has no series key', async () => { + fetchMock.restore().get('/api/tasks', []).get('/api/tasks/no-series-task', { + config: { rss: 'http://x.com' }, + name: 'no-series-task', + }).catch(); + const { queryByText, container } = renderWithWrapper( + , + ); + await wait(() => { + const el = getField(container, 'taskConfig'); + expect(el?.value).toContain('no-series-task'); + }, { timeout: 8000 }); + expect(queryByText('No series found in Source Task Config')).toBeInTheDocument(); + // The InputBase wrapping the Task Series Select should carry the Mui-disabled class + const seriesSelectRoot = container.querySelectorAll('.MuiSelect-root')[TASK_SERIES_SELECT_INDEX]; + expect(seriesSelectRoot.closest('.MuiInputBase-root')).toHaveClass('Mui-disabled'); + expect(getField(container, 'extras')).toBeInTheDocument(); + }); + }); + + // ------------------------------------------------------------------------- + // Auto-update: Backfill Task Name + // ------------------------------------------------------------------------- + + describe('Auto-update: Backfill Task Name', () => { + it('does NOT update taskName when auto-update is unchecked', async () => { + const { container } = renderWithWrapper(); + await waitForConfig(container); + + // Uncheck auto-update (starts checked after config loads) + fireEvent.click(container.querySelectorAll('input[type="checkbox"]')[0] as HTMLElement); + await pickSeriesOption(container, 'Breaking Bad'); + + expect(getField(container, 'taskName')?.value).toBe('test-task-backfill'); + }); + + it('updates taskName when auto-update is checked before series selection', async () => { + const { container } = renderWithWrapper(); + await waitForConfig(container); + + // auto-update checkbox is already checked after config loads + await pickSeriesOption(container, 'Breaking Bad'); + + await wait(() => + expect(getField(container, 'taskName')?.value).toBe('test-task-breaking-bad-backfill'), + ); + }); + + it('updates taskName immediately when auto-update is checked after series already selected', async () => { + const { container } = renderWithWrapper(); + await waitForConfig(container); + + // Uncheck so picking series does not immediately update taskName + fireEvent.click(container.querySelectorAll('input[type="checkbox"]')[0] as HTMLElement); + await pickSeriesOption(container, 'The Wire'); + // Recheck — should immediately update taskName with the already-selected series + fireEvent.click(container.querySelectorAll('input[type="checkbox"]')[0] as HTMLElement); + + await wait(() => + expect(getField(container, 'taskName')?.value).toBe('test-task-the-wire-backfill'), + ); + }); + + it('does not update taskName when auto-update is toggled off with no series selected', async () => { + const { container } = renderWithWrapper(); + await waitForConfig(container); + + // Click to uncheck (starts checked after config loads); no series selected — taskName unchanged + const checkbox = container.querySelectorAll('input[type="checkbox"]')[0] as HTMLInputElement; + fireEvent.click(checkbox); + + expect(getField(container, 'taskName')?.value).toBe('test-task-backfill'); + }); + + it('updates taskName to extras-only when auto-update is on with no series selected', async () => { + const { container } = renderWithWrapper(); + await waitForConfig(container); + + // auto-update checkbox is already checked after config loads + fireEvent.change(getField(container, 'extras')!, { target: { value: 'S01' } }); + + await wait(() => + expect(getField(container, 'taskName')?.value).toBe('test-task-s01-backfill'), + ); + }); + + it('updates taskName when extras changes while auto-update is on and series is already selected', async () => { + const { container } = renderWithWrapper(); + await waitForConfig(container); + + // auto-update checkbox is already checked after config loads + await pickSeriesOption(container, 'Breaking Bad'); + await wait(() => + expect(getField(container, 'taskName')?.value).toBe('test-task-breaking-bad-backfill'), + ); + + fireEvent.change(getField(container, 'extras')!, { target: { value: 'S01' } }); + + await wait(() => + expect(getField(container, 'taskName')?.value).toBe('test-task-breaking-bad-s01-backfill'), + ); + }); + + it('updates taskName immediately when auto-update is toggled on after both series and extras are already set', async () => { + const { container } = renderWithWrapper(); + await waitForConfig(container); + + // Uncheck auto-update first so picking series/extras does not immediately update taskName + fireEvent.click(container.querySelectorAll('input[type="checkbox"]')[0] as HTMLElement); + + await pickSeriesOption(container, 'Breaking Bad'); + fireEvent.change(getField(container, 'extras')!, { target: { value: 'S01' } }); + + // Auto-update was off during both inputs — task name should still be the default + expect(getField(container, 'taskName')?.value).toBe('test-task-backfill'); + + fireEvent.click(container.querySelectorAll('input[type="checkbox"]')[0] as HTMLElement); + + await wait(() => + expect(getField(container, 'taskName')?.value).toBe('test-task-breaking-bad-s01-backfill'), + ); + }); + }); + + // ------------------------------------------------------------------------- + // Auto-update: RSS URL + // ------------------------------------------------------------------------- + + describe('Auto-update: RSS URL', () => { + it('shows a snackbar when the disabled auto-update area is clicked', async () => { + const { container, queryByText } = renderWithWrapper( + , + ); + await waitForConfig(container); + + // The auto-update Box for RSS is the second checkbox row; click its container + const checkboxes = container.querySelectorAll('input[type="checkbox"]'); + const rssCheckboxContainer = checkboxes[1]?.closest('[role]') ?? checkboxes[1]?.parentElement?.parentElement; + if (rssCheckboxContainer) { + fireEvent.click(rssCheckboxContainer as HTMLElement); + } + + await wait(() => + expect(queryByText(/Please select the Query Param/)).toBeInTheDocument(), + ); + }); + + it('updates rssBackfillUrl when the selected query param is changed', async () => { + const { container } = renderWithWrapper(); + await waitForConfig(container); + + await pickSelectOption(container, QUERY_PARAM_SELECT_INDEX, 'q'); + await pickSeriesOption(container, 'Breaking Bad'); + await wait(() => + expect(getField(container, 'rssBackfillUrl')?.value).toContain('q=base+Breaking+Bad'), + ); + + // Switch param from 'q' to 'cat' — series should move to the new param + await pickSelectOption(container, QUERY_PARAM_SELECT_INDEX, 'cat'); + await wait(() => { + const url = getField(container, 'rssBackfillUrl')?.value ?? ''; + expect(url).toContain('cat=1+Breaking+Bad'); + expect(url).not.toContain('q=base+Breaking+Bad'); + }); + }); + + it('updates rssBackfillUrl immediately when query param is selected after series is already set', async () => { + const { container } = renderWithWrapper(); + await waitForConfig(container); + + // Pick series first — encodedSeriesName is now populated + await pickSeriesOption(container, 'Breaking Bad'); + + // Now select the query param — auto-update checkbox checks itself and + // should immediately update rssBackfillUrl using the already-set series + await pickSelectOption(container, QUERY_PARAM_SELECT_INDEX, 'q'); + + await wait(() => + expect(getField(container, 'rssBackfillUrl')?.value).toContain('q=base+Breaking+Bad'), + ); + }); + + it('updates rssBackfillUrl to reflect combined series and extras when auto-update is enabled', async () => { + const { container } = renderWithWrapper(); + await waitForConfig(container); + + // Pick query param 'q' — auto-update checkbox is checked automatically on first selection + await pickSelectOption(container, QUERY_PARAM_SELECT_INDEX, 'q'); + + // Select a series — encodedSeriesName becomes 'Breaking+Bad' + await pickSeriesOption(container, 'Breaking Bad'); + await wait(() => + expect(getField(container, 'rssBackfillUrl')?.value).toContain('q=base+Breaking+Bad'), + ); + + // Type extras — encodedSeriesName becomes 'Breaking+Bad-S01' + fireEvent.change(getField(container, 'extras')!, { target: { value: 'S01' } }); + await wait(() => + expect(getField(container, 'rssBackfillUrl')?.value).toContain('q=base+Breaking+Bad-S01'), + ); + }); + }); + + // ------------------------------------------------------------------------- + // Backfill Task Config auto-computation + // ------------------------------------------------------------------------- + + describe('Backfill Task Config auto-computation', () => { + it('reflects updated rssBackfillUrl in backfillTaskConfig', async () => { + const { container } = renderWithWrapper(); + await waitForConfig(container); + + fireEvent.change(getField(container, 'rssBackfillUrl')!, { + target: { value: 'http://new.example.com/rss' }, + }); + + await wait(() => { + const config = getField(container, 'backfillTaskConfig')?.value ?? ''; + expect(config).toContain('http://new.example.com/rss'); + }); + }); + + it('reflects updated taskName in backfillTaskConfig', async () => { + const { container } = renderWithWrapper(); + await waitForConfig(container); + + fireEvent.change(getField(container, 'taskName')!, { + target: { value: 'custom-task-name' }, + }); + + await wait(() => { + const config = getField(container, 'backfillTaskConfig')?.value ?? ''; + expect(config).toContain('custom-task-name'); + }); + }); + }); + + // ------------------------------------------------------------------------- + // Form validation + // ------------------------------------------------------------------------- + + describe('form validation', () => { + it('does not call createTask when required fields are missing', async () => { + fetchMock.restore().get('/api/tasks', []).get('/api/tasks/test-task', taskConfig).catch(); + const { container, getByText } = renderWithWrapper( + , + ); + await wait(() => expect(getField(container, 'taskConfig')).not.toBeNull()); + + const button = getByText('Backfill').closest('button')!; + fireEvent.click(button); + + await wait(() => { + expect(fetchMock.called('/api/tasks', { method: 'post' })).toBe(false); + }); + }); + }); + + // ------------------------------------------------------------------------- + // Submit: happy path (order-of-actions) + // ------------------------------------------------------------------------- + + describe('submit: happy path', () => { + it('calls create, execute, and delete APIs in order and logs each step', async () => { + const { container, getByText } = renderWithWrapper( + , + ); + await waitForConfig(container); + + fireEvent.change(getField(container, 'rssBackfillUrl')!, { + target: { value: 'http://backfill.example.com/rss' }, + }); + + fireEvent.click(getByText('Backfill').closest('button')!); + + await wait(() => { + const log = getField(container, 'executionLog')?.value ?? ''; + expect(log).toContain("Creating task 'test-task-backfill'..."); + expect(log).toContain("Task 'test-task-backfill' created."); + expect(log).toContain("Executing task 'test-task-backfill'..."); + }); + + expect(mockConnect).toHaveBeenCalledWith( + expect.objectContaining({ tasks: ['test-task-backfill'] }), + ); + + // Simulate stream completion + act(() => { capturedDoneCallback?.(); }); + + await wait(() => { + const log = getField(container, 'executionLog')?.value ?? ''; + expect(log).toContain("Deleting task 'test-task-backfill'..."); + expect(log).toContain("Task 'test-task-backfill' deleted."); + }); + }); + + it('passes the correct config body to createTask', async () => { + const { container, getByText } = renderWithWrapper( + , + ); + await waitForConfig(container); + + fireEvent.change(getField(container, 'rssBackfillUrl')!, { + target: { value: 'http://backfill.example.com/rss' }, + }); + fireEvent.click(getByText('Backfill').closest('button')!); + + await wait(() => expect(fetchMock.called('/api/tasks', { method: 'post' })).toBe(true)); + + const createCalls = fetchMock + .calls() + .filter(([url, opts]) => url === '/api/tasks' && (opts as RequestInit)?.method === 'post'); + const body = JSON.parse((createCalls[0][1] as RequestInit).body as string); + expect(body.name).toBe('test-task-backfill'); + expect(body.config.rss.url).toBe('http://backfill.example.com/rss'); + }); + + it('appends progress events to the execution log', async () => { + const { container, getByText } = renderWithWrapper( + , + ); + await waitForConfig(container); + fireEvent.change(getField(container, 'rssBackfillUrl')!, { + target: { value: 'http://backfill.example.com/rss' }, + }); + fireEvent.click(getByText('Backfill').closest('button')!); + await wait(() => expect(getField(container, 'executionLog')?.value).toContain("Executing")); + + act(() => { + capturedNodeHandlers['{progress}']?.({ + progress: { phase: 'input', plugin: 'rss' }, + }); + }); + + await wait(() => { + expect(getField(container, 'executionLog')?.value).toContain('[input] rss'); + }); + }); + + it('appends a non-aborted summary to the execution log', async () => { + const { container, getByText } = renderWithWrapper( + , + ); + await waitForConfig(container); + fireEvent.change(getField(container, 'rssBackfillUrl')!, { + target: { value: 'http://backfill.example.com/rss' }, + }); + fireEvent.click(getByText('Backfill').closest('button')!); + await wait(() => expect(getField(container, 'executionLog')?.value).toContain("Executing")); + + act(() => { + capturedNodeHandlers['{summary}']?.({ + summary: { aborted: false, accepted: 3, rejected: 1, failed: 0 }, + }); + }); + + await wait(() => { + expect(getField(container, 'executionLog')?.value).toContain( + 'Summary: 3 accepted, 1 rejected, 0 failed', + ); + }); + }); + + it('appends an aborted summary to the execution log', async () => { + const { container, getByText } = renderWithWrapper( + , + ); + await waitForConfig(container); + fireEvent.change(getField(container, 'rssBackfillUrl')!, { + target: { value: 'http://backfill.example.com/rss' }, + }); + fireEvent.click(getByText('Backfill').closest('button')!); + await wait(() => expect(getField(container, 'executionLog')?.value).toContain("Executing")); + + act(() => { + capturedNodeHandlers['{summary}']?.({ + summary: { aborted: true, abortReason: 'task failed to start' }, + }); + }); + + await wait(() => { + expect(getField(container, 'executionLog')?.value).toContain( + 'Aborted: task failed to start', + ); + }); + }); + }); + + // ------------------------------------------------------------------------- + // Submit: custom task name deletion fix + // ------------------------------------------------------------------------- + + describe('submit: custom task name is used for deletion', () => { + it('deletes using the edited taskName, not the default', async () => { + fetchMock + .restore() + .get('/api/tasks', []) + .get('/api/tasks/test-task', taskConfig) + .post('/api/tasks', 200) + .delete('/api/tasks/custom-name', 204) + .catch(); + + const { container, getByText } = renderWithWrapper( + , + ); + await waitForConfig(container); + + fireEvent.change(getField(container, 'taskName')!, { + target: { value: 'custom-name' }, + }); + fireEvent.change(getField(container, 'rssBackfillUrl')!, { + target: { value: 'http://backfill.example.com/rss' }, + }); + fireEvent.click(getByText('Backfill').closest('button')!); + + await wait(() => expect(getField(container, 'executionLog')?.value).toContain("Executing task 'custom-name'...")); + + act(() => { capturedDoneCallback?.(); }); + + await wait(() => { + const log = getField(container, 'executionLog')?.value ?? ''; + expect(log).toContain("Deleting task 'custom-name'..."); + expect(log).toContain("Task 'custom-name' deleted."); + expect(fetchMock.called('/api/tasks/custom-name', { method: 'delete' })).toBe(true); + expect(fetchMock.called('/api/tasks/test-task-backfill', { method: 'delete' })).toBe(false); + }); + }); + }); + + // ------------------------------------------------------------------------- + // Submit: error handling + // ------------------------------------------------------------------------- + + describe('submit: error handling', () => { + it('logs the error and does not connect when createTask fails', async () => { + fetchMock + .restore() + .get('/api/tasks', []) + .get('/api/tasks/test-task', taskConfig) + .post('/api/tasks', { status: 409, body: { message: 'Conflict' } }) + .catch(); + + const { container, getByText } = renderWithWrapper( + , + ); + await waitForConfig(container); + + fireEvent.change(getField(container, 'rssBackfillUrl')!, { + target: { value: 'http://backfill.example.com/rss' }, + }); + fireEvent.click(getByText('Backfill').closest('button')!); + + await wait(() => { + const log = getField(container, 'executionLog')?.value ?? ''; + expect(log).toContain('Error creating task:'); + }); + + expect(mockConnect).not.toHaveBeenCalled(); + }); + + it('logs "Unknown error" when createTask response carries no message', async () => { + jest.spyOn(backfillHooks, 'useCreateTask').mockReturnValue([ + { loading: false }, + jest.fn().mockResolvedValue({ ok: false, error: undefined }), + ] as any); + + const { container, getByText } = renderWithWrapper( + , + ); + await waitForConfig(container); + + fireEvent.change(getField(container, 'rssBackfillUrl')!, { + target: { value: 'http://backfill.example.com/rss' }, + }); + fireEvent.click(getByText('Backfill').closest('button')!); + + await wait(() => { + expect(getField(container, 'executionLog')?.value).toContain('Unknown error'); + }); + }); + + it('logs "Task execution failed." and still calls deleteTask when stream fails', async () => { + const { container, getByText } = renderWithWrapper( + , + ); + await waitForConfig(container); + + fireEvent.change(getField(container, 'rssBackfillUrl')!, { + target: { value: 'http://backfill.example.com/rss' }, + }); + fireEvent.click(getByText('Backfill').closest('button')!); + + await wait(() => expect(getField(container, 'executionLog')?.value).toContain("Executing")); + + act(() => { capturedFailCallback?.(); }); + + await wait(() => { + const log = getField(container, 'executionLog')?.value ?? ''; + expect(log).toContain('Task execution failed.'); + expect(fetchMock.called('/api/tasks/test-task-backfill', { method: 'delete' })).toBe(true); + }); + }); + + it('logs a delete error when deleteTask fails', async () => { + fetchMock + .restore() + .get('/api/tasks', []) + .get('/api/tasks/test-task', taskConfig) + .post('/api/tasks', 200) + .delete('/api/tasks/test-task-backfill', { status: 404, body: { message: 'Not Found' } }) + .catch(); + + const { container, getByText } = renderWithWrapper( + , + ); + await waitForConfig(container); + + fireEvent.change(getField(container, 'rssBackfillUrl')!, { + target: { value: 'http://backfill.example.com/rss' }, + }); + fireEvent.click(getByText('Backfill').closest('button')!); + + await wait(() => expect(getField(container, 'executionLog')?.value).toContain("Executing")); + + act(() => { capturedDoneCallback?.(); }); + + await wait(() => { + expect(getField(container, 'executionLog')?.value).toContain('Error deleting task:'); + }); + }); + }); +}); diff --git a/src/plugins/backfill/Backfill.tsx b/src/plugins/backfill/Backfill.tsx new file mode 100644 index 00000000..3b83a63a --- /dev/null +++ b/src/plugins/backfill/Backfill.tsx @@ -0,0 +1,581 @@ +import React, { FC, useState, useEffect, useCallback, useRef } from 'react'; +import { useLocation } from 'react-router-dom'; +import { Formik, Form, Field, useFormikContext } from 'formik'; +import { + Box, + Button, + Checkbox, + FormControl, + IconButton, + MenuItem, + Paper, + Select, + Theme, + TextField as MuiTextField, + Snackbar, + Tooltip, + Typography, +} from '@material-ui/core'; +import { DragHandle, ExpandLess, ExpandMore } from '@material-ui/icons'; +import { css } from '@emotion/core'; +import YAML from 'yaml'; +import { useInjectPageTitle } from 'core/layout/AppBar/hooks'; +import { useFlexgetStream } from 'core/api'; +import { Method, camelize } from 'utils/fetch'; +import TextField from 'common/inputs/formik/TextField'; +import { useGetTaskConfig, useCreateTask, useDeleteTask } from './hooks'; +import { FormValues } from './types'; +import { + replaceRssUrl, + replaceTaskName, + extractRssUrl, + extractTaskName, + extractSeriesNames, + appendToQueryParam, + extractQueryParamNames, + buildEncodedSeriesName, + buildAutoTaskName, + validate, +} from './utils'; + +// --------------------------------------------------------------------------- +// Styles +// --------------------------------------------------------------------------- + +const wrapper = (theme: Theme) => css` + margin: ${theme.typography.pxToRem(theme.spacing(2))}; + padding: ${theme.typography.pxToRem(theme.spacing(3))}; + display: grid; + grid-template-columns: max-content 1fr; + column-gap: ${theme.typography.pxToRem(theme.spacing(3))}; + row-gap: ${theme.typography.pxToRem(theme.spacing(3))}; + align-items: start; + + .MuiOutlinedInput-multiline { + padding: 12px 14px; + } + .MuiOutlinedInput-input:not(.MuiOutlinedInput-inputMultiline) { + padding-top: 12px; + padding-bottom: 12px; + } +`; + +const formLabel = (theme: Theme) => css` + white-space: nowrap; + color: ${theme.palette.text.secondary}; + font-size: ${theme.typography.body1.fontSize}; + line-height: 1.5; + padding-top: ${theme.typography.pxToRem(11)}; +`; + +const inputRow = (theme: Theme) => css` + display: flex; + align-items: flex-start; + gap: ${theme.typography.pxToRem(theme.spacing(2))}; +`; + +// --------------------------------------------------------------------------- +// Shared primitives +// --------------------------------------------------------------------------- + +// Returns true on exactly the one render where `value` first becomes non-empty. +const useFirstPopulation = (value: string): boolean => { + const prevRef = useRef(''); + const isFirst = !prevRef.current && !!value; + prevRef.current = value; + return isFirst; +}; + +// Floating label above an input control, used in the horizontal input rows. +interface LabeledBoxProps extends Omit, 'css'> { + label: string; +} +const LabeledBox: FC = ({ label, style, children, ...rest }) => ( + + + {label} + + {children} + +); + +// One row of the two-column label/content grid. +const FormRow: FC<{ label: React.ReactNode; spacing?: boolean }> = ({ label, children, spacing }) => ( + <> +
{label}
+
{children}
+ +); + +// --------------------------------------------------------------------------- +// Resize hook + handle +// --------------------------------------------------------------------------- + +const useTextareaResize = () => { + const inputRef = useRef(null); + const [dragHeight, setDragHeight] = useState(undefined); + + const onResizeMouseDown = useCallback((e: React.MouseEvent) => { + e.preventDefault(); + const startY = e.clientY; + const startH = inputRef.current?.clientHeight ?? 80; + const onMove = (ev: MouseEvent) => { + setDragHeight(Math.max(startH + ev.clientY - startY, 36)); + }; + const onUp = () => { + document.removeEventListener('mousemove', onMove); + document.removeEventListener('mouseup', onUp); + }; + document.addEventListener('mousemove', onMove); + document.addEventListener('mouseup', onUp); + }, []); + + const resetDragHeight = useCallback(() => setDragHeight(undefined), []); + + return { inputRef, dragHeight, onResizeMouseDown, resetDragHeight }; +}; + +const ResizeHandle: FC<{ onMouseDown: (e: React.MouseEvent) => void }> = ({ onMouseDown }) => ( + + + +); + +// --------------------------------------------------------------------------- +// ExpandableTextarea — shared by Source Task Config and Backfill Task Config +// --------------------------------------------------------------------------- + +interface ExpandableTextareaProps { + label: string; + name: string; + onChange?: (e: React.ChangeEvent) => void; +} + +const ExpandableTextarea: FC = ({ label, name, onChange }) => { + const [expanded, setExpanded] = useState(false); + const { inputRef, dragHeight, onResizeMouseDown, resetDragHeight } = useTextareaResize(); + + return ( + + {label} + { setExpanded(v => !v); resetDragHeight(); }}> + {expanded ? : } + + + } + > + + + {expanded && } + + + ); +}; + +// --------------------------------------------------------------------------- +// Form field components +// --------------------------------------------------------------------------- + +const SourceTaskConfigField: FC = () => { + const { handleChange, setFieldValue } = useFormikContext(); + return ( + { + handleChange(e); + setFieldValue('sourceBackfillUrl', extractRssUrl(e.target.value)); + setFieldValue('sourceTaskName', extractTaskName(e.target.value)); + }} + /> + ); +}; + +const BackfillTaskConfigField: FC = () => { + const { values, setFieldValue } = useFormikContext(); + useEffect(() => { + const withRss = replaceRssUrl(values.taskConfig, values.rssBackfillUrl); + setFieldValue('backfillTaskConfig', replaceTaskName(withRss, values.taskName)); + }, [values.taskConfig, values.rssBackfillUrl, values.taskName, setFieldValue]); + return ; +}; + +const TaskSeriesField: FC = () => { + const { values, setFieldValue, handleChange } = useFormikContext(); + const seriesNames = extractSeriesNames(values.taskConfig).sort((a, b) => a.localeCompare(b)); + const isFirstSourceTaskName = useFirstPopulation(values.sourceTaskName); + + useEffect(() => { + if (!values.sourceTaskName) return; + setFieldValue( + 'taskName', + buildAutoTaskName( + values.sourceTaskName, + values.autoUpdateTaskName ? values.selectedSeries : '', + values.autoUpdateTaskName ? values.extras : '', + ), + ); + if (isFirstSourceTaskName) { + setFieldValue('autoUpdateTaskName', true); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [values.sourceTaskName]); + + const handleSeriesChange = (e: React.ChangeEvent<{ value: unknown }>) => { + const value = e.target.value as string; + setFieldValue('selectedSeries', value); + if (values.autoUpdateTaskName) { + setFieldValue('taskName', buildAutoTaskName(values.sourceTaskName, value, values.extras)); + } + }; + + return ( + +
+ + + + + + + ) => { + handleChange(e); + if (values.autoUpdateTaskName) { + setFieldValue('taskName', buildAutoTaskName(values.sourceTaskName, values.selectedSeries, e.target.value)); + } + }} + /> + +
+ + ); +}; + +const BackfillTaskNameRow: FC = () => { + const { values, setFieldValue } = useFormikContext(); + + const handleAutoUpdateChange = (e: React.ChangeEvent) => { + const checked = e.target.checked; + setFieldValue('autoUpdateTaskName', checked); + if (checked && (values.selectedSeries || values.extras)) { + setFieldValue('taskName', buildAutoTaskName(values.sourceTaskName, values.selectedSeries, values.extras)); + } + }; + + return ( + +
+ + + + + + + + +
+ + ); +}; + +const RssBackfillUrlField: FC = () => { + const { handleChange, setFieldValue, values } = useFormikContext(); + const [selectedParam, setSelectedParam] = useState(''); + const [autoUpdate, setAutoUpdate] = useState(false); + const [disabledSnackOpen, setDisabledSnackOpen] = useState(false); + const paramNames = extractQueryParamNames(values.rssBackfillUrl); + const encodedSeriesName = buildEncodedSeriesName(values.selectedSeries, values.extras); + const isFirstParamSelection = useFirstPopulation(selectedParam); + + useEffect(() => { + if (!autoUpdate || !selectedParam || !encodedSeriesName) return; + setFieldValue('rssBackfillUrl', appendToQueryParam(values.sourceBackfillUrl, selectedParam, encodedSeriesName)); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [values.selectedSeries, values.extras]); + + useEffect(() => { + if (autoUpdate && selectedParam && encodedSeriesName) { + setFieldValue('rssBackfillUrl', appendToQueryParam(values.sourceBackfillUrl, selectedParam, encodedSeriesName)); + } else { + setFieldValue('rssBackfillUrl', values.sourceBackfillUrl); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [values.sourceBackfillUrl]); + + useEffect(() => { + if (isFirstParamSelection) { + setAutoUpdate(true); + } + if ((isFirstParamSelection || autoUpdate) && selectedParam && encodedSeriesName) { + setFieldValue('rssBackfillUrl', appendToQueryParam(values.sourceBackfillUrl, selectedParam, encodedSeriesName)); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [selectedParam]); + + const handleAutoUpdateChange = (e: React.ChangeEvent) => { + const checked = e.target.checked; + setAutoUpdate(checked); + if (checked && selectedParam && encodedSeriesName) { + setFieldValue('rssBackfillUrl', appendToQueryParam(values.sourceBackfillUrl, selectedParam, encodedSeriesName)); + } + }; + + return ( + + <> +
+ + ) => { + handleChange(e); + }} + /> + + + + + + + + setDisabledSnackOpen(true) : undefined} + > + + + +
+ setDisabledSnackOpen(false)} + message="Please select the Query Param that will be updated, to enable automatic updating" + /> + + + ); +}; + +// --------------------------------------------------------------------------- +// Root component +// --------------------------------------------------------------------------- + +const Backfill: FC = () => { + const taskParam = new URLSearchParams(useLocation().search).get('task'); + useInjectPageTitle(`Backfill Task: ${taskParam ?? 'Unknown'}`); + + const taskName = taskParam ? `${taskParam}-backfill` : ''; + const { config: taskConfig } = useGetTaskConfig(taskParam ?? ''); + const [executionLog, setExecutionLog] = useState(' '); + const pendingDelete = useRef(false); + const logRef = useRef(null); + + useEffect(() => { + if (logRef.current) { + logRef.current.scrollTop = logRef.current.scrollHeight; + } + }, [executionLog]); + + const [deleteTaskName, setDeleteTaskName] = useState(taskName); + const [, createTask] = useCreateTask(); + const [, deleteTask] = useDeleteTask(deleteTaskName); + const [{ stream }, { connect }] = useFlexgetStream('/tasks/execute', Method.Post); + + const appendLog = useCallback((line: string) => { + setExecutionLog(prev => `${prev}${line}\n`); + }, []); + + const runDelete = useCallback(() => { + appendLog(`Deleting task '${deleteTaskName}'...`); + deleteTask().then(resp => { + appendLog( + resp.ok + ? `Task '${deleteTaskName}' deleted.` + : `Error deleting task: ${resp.error?.message ?? 'Unknown error'}`, + ); + }); + }, [appendLog, deleteTask, deleteTaskName]); + + const runDeleteRef = useRef(runDelete); + useEffect(() => { + runDeleteRef.current = runDelete; + }, [runDelete]); + + useEffect(() => { + if (!stream) return; + stream + .node('{progress}', (e: any) => { + const ev = camelize(e) as any; + appendLog(`[${ev.progress.phase}] ${ev.progress.plugin}`); + }) + .node('{summary}', (e: any) => { + const ev = camelize(e) as any; + if (ev.summary.aborted) { + appendLog(`Aborted: ${ev.summary.abortReason ?? 'unknown reason'}`); + } else { + appendLog( + `Summary: ${ev.summary.accepted} accepted, ${ev.summary.rejected} rejected, ${ev.summary.failed} failed`, + ); + } + }) + .done(() => { + if (!pendingDelete.current) return; + pendingDelete.current = false; + runDeleteRef.current(); + }) + .fail(() => { + if (!pendingDelete.current) return; + pendingDelete.current = false; + appendLog('Task execution failed.'); + runDeleteRef.current(); + }); + }, [stream, appendLog]); + + const initialValues: FormValues = { + taskName, + autoUpdateTaskName: false, + rssBackfillUrl: taskParam ? extractRssUrl(taskConfig) : '', + sourceBackfillUrl: taskParam ? extractRssUrl(taskConfig) : '', + taskConfig: taskParam ? taskConfig : 'No task selected', + selectedSeries: '', + backfillTaskConfig: '', + extras: '', + sourceTaskName: taskParam ? extractTaskName(taskConfig) : '', + }; + + return ( + { + setExecutionLog(''); + setDeleteTaskName(values.taskName); + const updatedParsed = YAML.parse(values.backfillTaskConfig); + + appendLog(`Creating task '${values.taskName}'...`); + const createResp = await createTask( + values.taskName, + updatedParsed.config ?? updatedParsed, + ); + if (!createResp.ok) { + appendLog(`Error creating task: ${createResp.error?.message ?? 'Unknown error'}`); + return; + } + appendLog(`Task '${values.taskName}' created.`); + appendLog(`Executing task '${values.taskName}'...`); + pendingDelete.current = true; + connect({ tasks: [values.taskName], progress: true, summary: true }); + }} + > +
+ + + + + + + + + + + + + + + +
+ + ); +}; + +export default Backfill; diff --git a/src/plugins/backfill/hooks.ts b/src/plugins/backfill/hooks.ts new file mode 100644 index 00000000..092719d0 --- /dev/null +++ b/src/plugins/backfill/hooks.ts @@ -0,0 +1,37 @@ +import { useEffect, useState, useCallback } from 'react'; +import YAML from 'yaml'; +import { useFlexgetAPI } from 'core/api'; +import { Method } from 'utils/fetch'; + +export const useGetTaskConfig = (taskName: string) => { + const [config, setConfig] = useState(''); + const [state, request] = useFlexgetAPI>( + `/tasks/${encodeURIComponent(taskName)}`, + ); + + const fetch = useCallback(async () => { + if (!taskName) return; + const resp = await request(); + if (resp.ok) { + setConfig(YAML.stringify(resp.data)); + } + }, [request, taskName]); + + useEffect(() => { + fetch(); + }, [fetch]); + + return { ...state, config }; +}; + +export const useCreateTask = () => { + const [state, request] = useFlexgetAPI>('/tasks', Method.Post); + const create = useCallback( + (name: string, config: Record) => request({ name, config }), + [request], + ); + return [state, create] as const; +}; + +export const useDeleteTask = (name: string) => + useFlexgetAPI(`/tasks/${encodeURIComponent(name)}`, Method.Delete); diff --git a/src/plugins/backfill/index.ts b/src/plugins/backfill/index.ts new file mode 100644 index 00000000..d7f69220 --- /dev/null +++ b/src/plugins/backfill/index.ts @@ -0,0 +1,17 @@ +import { lazy } from 'react'; +import UpdateIcon from '@material-ui/icons/Update'; +import { registerPlugin } from 'core/plugins/registry'; + +export default () => + registerPlugin('/backfill', { + component: lazy( + () => + import( + /* webpackChunkName: 'BackfillPlugin' */ + 'plugins/backfill/Backfill' + ), + ), + displayName: 'Backfill', + icon: UpdateIcon, + hidden: true, + }); diff --git a/src/plugins/backfill/types.ts b/src/plugins/backfill/types.ts new file mode 100644 index 00000000..cdd72541 --- /dev/null +++ b/src/plugins/backfill/types.ts @@ -0,0 +1,11 @@ +export interface FormValues { + taskName: string; + autoUpdateTaskName: boolean; + rssBackfillUrl: string; + sourceBackfillUrl: string; + taskConfig: string; + selectedSeries: string; + backfillTaskConfig: string; + extras: string; + sourceTaskName: string; +} diff --git a/src/plugins/backfill/utils.spec.ts b/src/plugins/backfill/utils.spec.ts new file mode 100644 index 00000000..6bd48bda --- /dev/null +++ b/src/plugins/backfill/utils.spec.ts @@ -0,0 +1,256 @@ +import YAML from 'yaml'; +import { + toKebabCase, + extractRssUrl, + replaceRssUrl, + replaceTaskName, + extractSeriesNames, + appendToQueryParam, + extractQueryParamNames, + validate, +} from './utils'; +import { FormValues } from './types'; + +// --------------------------------------------------------------------------- +// toKebabCase +// --------------------------------------------------------------------------- +describe('toKebabCase', () => { + it('lowercases and hyphenates words', () => { + expect(toKebabCase('Breaking Bad')).toBe('breaking-bad'); + }); + + it('strips leading and trailing punctuation', () => { + expect(toKebabCase('--Show Name--')).toBe('show-name'); + }); + + it('collapses consecutive non-alphanumeric chars into one hyphen', () => { + expect(toKebabCase('Mr. Robot!')).toBe('mr-robot'); + }); + + it('leaves an already-kebab string unchanged', () => { + expect(toKebabCase('already-kebab')).toBe('already-kebab'); + }); + + it('handles numeric characters', () => { + expect(toKebabCase('Show 2049')).toBe('show-2049'); + }); +}); + +// --------------------------------------------------------------------------- +// extractRssUrl +// --------------------------------------------------------------------------- +describe('extractRssUrl', () => { + it('extracts a direct string rss key', () => { + const yaml = YAML.stringify({ rss: 'http://direct.com/feed' }); + expect(extractRssUrl(yaml)).toBe('http://direct.com/feed'); + }); + + it('extracts rss.url from an object rss key', () => { + const yaml = YAML.stringify({ rss: { url: 'http://obj.com/feed', other: true } }); + expect(extractRssUrl(yaml)).toBe('http://obj.com/feed'); + }); + + it('extracts from nested config.rss string', () => { + const yaml = YAML.stringify({ config: { rss: 'http://nested.com/feed' }, name: 'task' }); + expect(extractRssUrl(yaml)).toBe('http://nested.com/feed'); + }); + + it('extracts from nested config.rss.url', () => { + const yaml = YAML.stringify({ + config: { rss: { url: 'http://nested-obj.com/feed' } }, + name: 'task', + }); + expect(extractRssUrl(yaml)).toBe('http://nested-obj.com/feed'); + }); + + it('returns empty string when no rss key', () => { + expect(extractRssUrl(YAML.stringify({ name: 'no-rss' }))).toBe(''); + }); + + it('returns empty string for malformed YAML', () => { + expect(extractRssUrl(': bad: yaml: {')).toBe(''); + }); + + it('returns empty string for empty input', () => { + expect(extractRssUrl('')).toBe(''); + }); +}); + +// --------------------------------------------------------------------------- +// replaceRssUrl +// --------------------------------------------------------------------------- +describe('replaceRssUrl', () => { + it('replaces a direct string rss value', () => { + const input = YAML.stringify({ rss: 'http://old.com', name: 'task' }); + const result = YAML.parse(replaceRssUrl(input, 'http://new.com')); + expect(result.rss).toBe('http://new.com'); + expect(result.name).toBe('task'); + }); + + it('replaces rss.url in object form', () => { + const input = YAML.stringify({ rss: { url: 'http://old.com', ttl: 30 } }); + const result = YAML.parse(replaceRssUrl(input, 'http://new.com')); + expect(result.rss.url).toBe('http://new.com'); + expect(result.rss.ttl).toBe(30); + }); + + it('replaces nested config.rss string', () => { + const input = YAML.stringify({ config: { rss: 'http://old.com' }, name: 'task' }); + const result = YAML.parse(replaceRssUrl(input, 'http://new.com')); + expect(result.config.rss).toBe('http://new.com'); + }); + + it('replaces nested config.rss.url', () => { + const input = YAML.stringify({ config: { rss: { url: 'http://old.com' } }, name: 'task' }); + const result = YAML.parse(replaceRssUrl(input, 'http://new.com')); + expect(result.config.rss.url).toBe('http://new.com'); + }); + + it('returns the original string unchanged for non-object YAML', () => { + expect(replaceRssUrl('just a string', 'http://new.com')).toBe('just a string'); + }); +}); + +// --------------------------------------------------------------------------- +// replaceTaskName +// --------------------------------------------------------------------------- +describe('replaceTaskName', () => { + it('replaces the name key', () => { + const input = YAML.stringify({ name: 'old-name', config: { rss: 'http://x.com' } }); + const result = YAML.parse(replaceTaskName(input, 'new-name')); + expect(result.name).toBe('new-name'); + expect(result.config.rss).toBe('http://x.com'); + }); + + it('returns original string when no name key present', () => { + const input = YAML.stringify({ config: { rss: 'http://x.com' } }); + expect(replaceTaskName(input, 'new-name')).toBe(input); + }); + + it('returns original string for non-object YAML', () => { + expect(replaceTaskName('plain string', 'new-name')).toBe('plain string'); + }); +}); + +// --------------------------------------------------------------------------- +// extractSeriesNames +// --------------------------------------------------------------------------- +describe('extractSeriesNames', () => { + it('extracts a flat array of string series', () => { + const yaml = YAML.stringify({ series: ['Breaking Bad', 'The Wire'] }); + expect(extractSeriesNames(yaml)).toEqual(expect.arrayContaining(['Breaking Bad', 'The Wire'])); + }); + + it('extracts series from array of single-key objects', () => { + const yaml = YAML.stringify({ series: [{ 'Show Name': { quality: 'hdtv' } }] }); + expect(extractSeriesNames(yaml)).toContain('Show Name'); + }); + + it('extracts from grouped object format', () => { + const yaml = YAML.stringify({ series: { group1: ['Foo', 'Bar'] } }); + expect(extractSeriesNames(yaml)).toEqual(expect.arrayContaining(['Foo', 'Bar'])); + }); + + it('extracts from nested config.series', () => { + const yaml = YAML.stringify({ config: { series: ['Nested Show'] }, name: 'task' }); + expect(extractSeriesNames(yaml)).toContain('Nested Show'); + }); + + it('returns empty array when no series key', () => { + expect(extractSeriesNames(YAML.stringify({ name: 'task' }))).toEqual([]); + }); + + it('returns empty array for malformed YAML', () => { + expect(extractSeriesNames(': { bad')).toEqual([]); + }); + + it('returns empty array for empty input', () => { + expect(extractSeriesNames('')).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +// appendToQueryParam +// --------------------------------------------------------------------------- +describe('appendToQueryParam', () => { + it('appends suffix to the named param value', () => { + expect(appendToQueryParam('http://x.com?q=base&cat=1', 'q', 'Foo')).toBe( + 'http://x.com?q=base+Foo&cat=1', + ); + }); + + it('leaves URL unchanged when param is not present', () => { + const url = 'http://x.com?other=1'; + expect(appendToQueryParam(url, 'q', 'Foo')).toBe(url); + }); + + it('only appends to the first matching param', () => { + const result = appendToQueryParam('http://x.com?q=a&q=b', 'q', 'X'); + expect(result).toBe('http://x.com?q=a+X&q=b'); + }); +}); + +// --------------------------------------------------------------------------- +// extractQueryParamNames +// --------------------------------------------------------------------------- +describe('extractQueryParamNames', () => { + it('returns all param names from a valid URL', () => { + expect(extractQueryParamNames('http://x.com?foo=1&bar=2')).toEqual(['foo', 'bar']); + }); + + it('returns empty array for URL with no query string', () => { + expect(extractQueryParamNames('http://x.com/path')).toEqual([]); + }); + + it('returns empty array for an invalid URL without throwing', () => { + expect(extractQueryParamNames('not a url')).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +// validate +// --------------------------------------------------------------------------- +describe('validate', () => { + const base: FormValues = { + taskName: 'my-task', + autoUpdateTaskName: false, + rssBackfillUrl: 'http://x.com', + sourceBackfillUrl: 'http://x.com', + taskConfig: 'name: my-task', + selectedSeries: '', + backfillTaskConfig: '', + extras: '', + sourceTaskName: '', + }; + + it('returns no errors when all required fields are present', () => { + expect(validate(base)).toEqual({}); + }); + + it('requires taskName', () => { + expect(validate({ ...base, taskName: '' })).toMatchObject({ + taskName: 'Task name is required', + }); + }); + + it('requires rssBackfillUrl', () => { + expect(validate({ ...base, rssBackfillUrl: '' })).toMatchObject({ + rssBackfillUrl: 'Backfill RSS URL is required', + }); + }); + + it('requires taskConfig', () => { + expect(validate({ ...base, taskConfig: '' })).toMatchObject({ + taskConfig: 'Task Config is required', + }); + }); + + it('returns all three errors when all required fields are empty', () => { + const errors = validate({ ...base, taskName: '', rssBackfillUrl: '', taskConfig: '' }); + expect(errors).toMatchObject({ + taskName: expect.any(String), + rssBackfillUrl: expect.any(String), + taskConfig: expect.any(String), + }); + }); +}); diff --git a/src/plugins/backfill/utils.ts b/src/plugins/backfill/utils.ts new file mode 100644 index 00000000..8553e7cb --- /dev/null +++ b/src/plugins/backfill/utils.ts @@ -0,0 +1,122 @@ +import YAML from 'yaml'; +import { FormValues } from './types'; + +export const toKebabCase = (str: string): string => + str + .toLowerCase() + .replace(/[^a-z0-9]+/g, '-') + .replace(/^-+|-+$/g, ''); + +const applyRssUrl = (obj: Record, newUrl: string): boolean => { + if (typeof obj.rss === 'string') { + obj.rss = newUrl; + return true; + } + if (obj.rss && typeof obj.rss === 'object' && 'url' in obj.rss) { + obj.rss.url = newUrl; + return true; + } + return false; +}; + +export const replaceRssUrl = (taskConfig: string, newUrl: string): string => { + const parsed = YAML.parse(taskConfig); + if (!parsed || typeof parsed !== 'object') return taskConfig; + if (!applyRssUrl(parsed, newUrl) && parsed.config && typeof parsed.config === 'object') { + applyRssUrl(parsed.config, newUrl); + } + return YAML.stringify(parsed); +}; + +export const replaceTaskName = (taskConfig: string, taskName: string): string => { + const parsed = YAML.parse(taskConfig); + if (!parsed || typeof parsed !== 'object' || !('name' in parsed)) return taskConfig; + parsed.name = taskName; + return YAML.stringify(parsed); +}; + +export const extractTaskName = (taskConfig: string): string => { + try { + const parsed = YAML.parse(taskConfig); + if (!parsed || typeof parsed !== 'object') return ''; + return typeof parsed.name === 'string' ? parsed.name : ''; + } catch { + return ''; + } +}; + +export const extractRssUrl = (taskConfig: string): string => { + try { + const parsed = YAML.parse(taskConfig); + if (!parsed || typeof parsed !== 'object') return ''; + if (typeof parsed.rss === 'string') return parsed.rss; + if (parsed.rss && typeof parsed.rss === 'object' && 'url' in parsed.rss) return parsed.rss.url; + if (parsed.config && typeof parsed.config === 'object') { + if (typeof parsed.config.rss === 'string') return parsed.config.rss; + if (parsed.config.rss && typeof parsed.config.rss === 'object' && 'url' in parsed.config.rss) + return parsed.config.rss.url; + } + } catch { + // ignore malformed YAML + } + return ''; +}; + +const extractSeriesEntry = (entry: any): string[] => { + if (typeof entry === 'string') return [entry]; + if (entry && typeof entry === 'object') return Object.keys(entry); + return []; +}; + +export const extractSeriesNames = (taskConfig: string): string[] => { + try { + const parsed = YAML.parse(taskConfig); + if (!parsed || typeof parsed !== 'object') return []; + const seriesConfig = parsed.config?.series ?? parsed.series; + if (!seriesConfig) return []; + if (Array.isArray(seriesConfig)) { + return seriesConfig.flatMap(extractSeriesEntry); + } + if (typeof seriesConfig === 'object') { + return Object.values(seriesConfig).flatMap((group: any) => + Array.isArray(group) ? group.flatMap(extractSeriesEntry) : [], + ); + } + } catch { + // ignore malformed YAML + } + return []; +}; + +export const buildAutoTaskName = (taskParam: string, series: string, extras: string): string => { + const combined = [series, extras].filter(Boolean).join('-'); + return combined + ? `${taskParam}-${toKebabCase(combined)}-backfill` + : `${taskParam}-backfill`; +}; + +export const buildEncodedSeriesName = (series: string, extras: string): string => { + const combined = [series, extras].filter(Boolean).join('-'); + return encodeURIComponent(combined).replace(/%20/g, '+'); +}; + +export const appendToQueryParam = (url: string, paramName: string, suffix: string): string => { + const regex = new RegExp(`([?&]${paramName}=)([^&]*)`); + return url.replace(regex, `$1$2+${suffix}`); +}; + +export const extractQueryParamNames = (url: string): string[] => { + try { + return Array.from(new URL(url).searchParams.keys()); + } catch { + return []; + } +}; + +export const validate = (values: FormValues): Partial => { + const errors: Partial = {}; + if (!values.taskName) errors.taskName = 'Task name is required'; + if (!values.rssBackfillUrl) errors.rssBackfillUrl = 'Backfill RSS URL is required'; + if (!values.taskConfig) errors.taskConfig = 'Task Config is required'; + return errors; +}; diff --git a/src/plugins/tasks/Latest.tsx b/src/plugins/tasks/Latest.tsx index 03f0e389..c802771a 100644 --- a/src/plugins/tasks/Latest.tsx +++ b/src/plugins/tasks/Latest.tsx @@ -1,7 +1,9 @@ import React, { FC, useMemo, useState } from 'react'; import { Formik } from 'formik'; import { useHistory, useRouteMatch } from 'react-router'; -import { CheckCircle, Error } from '@material-ui/icons'; +import { Link } from 'react-router-dom'; +import { IconButton, Tooltip } from '@material-ui/core'; +import { CheckCircle, Error, Update } from '@material-ui/icons'; import { useInjectPageTitle } from 'core/layout/AppBar/hooks'; import { Direction } from 'utils/query'; import { useContainer } from 'unstated-next'; @@ -54,6 +56,10 @@ const headers = [ id: SortByStatus.AbortReason, label: 'Abort Reason', }, + { + id: SortByStatus.Backfill, + label: '', + }, ]; const Latest: FC = () => { @@ -103,6 +109,18 @@ const Latest: FC = () => { ) : ( ), + [SortByStatus.Backfill]: ( + + ) => e.stopPropagation()} + > + + + + ), }, props: { onClick: () => push(`${url}/${id}`), diff --git a/src/plugins/tasks/TaskExecutions.tsx b/src/plugins/tasks/TaskExecutions.tsx index b1406309..081cf3a2 100644 --- a/src/plugins/tasks/TaskExecutions.tsx +++ b/src/plugins/tasks/TaskExecutions.tsx @@ -112,6 +112,7 @@ const TaskExecutions: FC = () => { ) : ( ), + [SortByStatus.Backfill]: undefined, }, }), ), diff --git a/src/plugins/tasks/types.ts b/src/plugins/tasks/types.ts index 71d27b50..6f0689ec 100644 --- a/src/plugins/tasks/types.ts +++ b/src/plugins/tasks/types.ts @@ -62,6 +62,7 @@ export const enum SortByStatus { Failed = 'failed', Succeeded = 'succeeded', AbortReason = 'abort_reason', + Backfill = 'backfill', } export interface TaskStatusOptions extends Partial {