From 77f6a187db364729c28388c8226bfb4ad0d664f4 Mon Sep 17 00:00:00 2001 From: Harsh16gupta Date: Fri, 31 Jul 2026 09:58:18 +0530 Subject: [PATCH 1/3] feat: refactor plugin architecture, webview state persistence, and native settings --- src/commands/applyChanges.ts | 31 ++++- src/commands/applyNotebooks.ts | 23 +++- src/commands/registerCommands.ts | 62 +++++++++ src/index.ts | 174 ++---------------------- src/panel/setupPanel.ts | 157 +++++++++++++++++++++ src/settings/registerSettings.ts | 162 ++++++++++++++++++++++ src/types/panel.ts | 4 +- src/webview/components/Navigation.tsx | 37 ----- src/webview/context/AppStateContext.tsx | 59 ++++++-- src/webview/context/useApplyState.ts | 86 +++++++----- src/webview/context/usePipelineState.ts | 14 +- src/webview/context/useSettingsState.ts | 5 + src/webview/pages/DashboardPage.tsx | 67 +++++---- src/webview/pages/SettingsPage.tsx | 26 ++-- src/webview/panel.css | 58 ++------ src/webview/panel.tsx | 12 +- 16 files changed, 631 insertions(+), 346 deletions(-) create mode 100644 src/commands/registerCommands.ts create mode 100644 src/panel/setupPanel.ts create mode 100644 src/settings/registerSettings.ts delete mode 100644 src/webview/components/Navigation.tsx diff --git a/src/commands/applyChanges.ts b/src/commands/applyChanges.ts index 190718b..3d2ba6a 100644 --- a/src/commands/applyChanges.ts +++ b/src/commands/applyChanges.ts @@ -30,6 +30,23 @@ export interface ChangeLogEntry { createdTagIds?: string[]; } +function formatChangeLogSummary(entry: ChangeLogEntry): string { + const date = new Date(entry.timestamp); + const dateStr = date.toLocaleString(undefined, { + year: 'numeric', + month: 'short', + day: 'numeric', + hour: '2-digit', + minute: '2-digit', + }); + const methodLabel = + entry.method === 'both' ? 'Notebooks & Tags' : entry.method === 'notebooks' ? 'Notebooks Only' : 'Tags Only'; + const noteCount = entry.notes?.length || 0; + const folderCount = entry.createdFolderIds?.length || 0; + const tagCount = entry.createdTagIds?.length || 0; + return `Applied: ${dateStr} | Method: ${methodLabel} | Modified Notes: ${noteCount} | Folders Created: ${folderCount} | Tags Created: ${tagCount}`; +} + export async function applyCategorizationChanges( options: ApplyOptions, notes: PanelNote[], @@ -37,7 +54,7 @@ export async function applyCategorizationChanges( clusterNames: { [clusterId: number]: string }, clusterTags: { [clusterId: number]: string[] }, setPanelState: (state: PanelMessage) => void, -) { +): Promise { try { setPanelState({ type: 'apply_status', text: 'Fetching existing folders and tags...' }); @@ -59,12 +76,15 @@ export async function applyCategorizationChanges( let folderMap: { [clusterId: number]: string } = {}; let uncategorizedFolderId = ''; if (options.method === 'notebooks' || options.method === 'both') { + const targetParentNotebook = + options.parentNotebookName || (await joplin.settings.value('categorization.parentNotebook')) || ''; const initFolders = await initializeClusterNotebooks( uniqueClusterIds, clusterNames, assignments, existingFoldersMap, createdFolderIds, + targetParentNotebook, ); folderMap = initFolders.folderMap; uncategorizedFolderId = initFolders.uncategorizedFolderId; @@ -108,7 +128,12 @@ export async function applyCategorizationChanges( noteTitle = noteObj.title || ''; noteBody = needsBody ? noteObj.body || '' : ''; } catch (fetchErr) { - log(`Error fetching note data for ${note.noteId}: ${fetchErr}`); + log(`Error fetching note data for ${note.noteId} (note may have been deleted): ${fetchErr}`); + setPanelState({ + type: 'apply_progress', + current: i + 1, + total, + }); continue; } @@ -166,6 +191,7 @@ export async function applyCategorizationChanges( createdTagIds, }; await joplin.settings.setValue('categorization.changeLog', JSON.stringify(changeLogEntry)); + await joplin.settings.setValue('categorization.changeLogSummary', formatChangeLogSummary(changeLogEntry)); setPanelState({ type: 'apply_complete' }); } catch (err) { @@ -229,6 +255,7 @@ export async function undoCategorizationChanges(setPanelState: (state: PanelMess // Clear change log await joplin.settings.setValue('categorization.changeLog', ''); + await joplin.settings.setValue('categorization.changeLogSummary', 'No categorization has been applied yet.'); setPanelState({ type: 'undo_complete' }); } catch (err) { diff --git a/src/commands/applyNotebooks.ts b/src/commands/applyNotebooks.ts index c8c18f3..5117379 100644 --- a/src/commands/applyNotebooks.ts +++ b/src/commands/applyNotebooks.ts @@ -50,13 +50,28 @@ export async function initializeClusterNotebooks( assignments: number[], existingFoldersMap: Map, createdFolderIds: string[], + parentNotebookName = '', ): Promise<{ folderMap: { [clusterId: number]: string }; uncategorizedFolderId: string }> { const folderMap: { [clusterId: number]: string } = {}; let uncategorizedFolderId = ''; + let rootParentFolderId = ''; + + const trimmedParent = parentNotebookName.trim(); + if (trimmedParent) { + const { id: parentFolderId, created } = await getOrCreateFolder(existingFoldersMap, trimmedParent, ''); + rootParentFolderId = parentFolderId; + if (created) { + createdFolderIds.push(parentFolderId); + } + } for (const clusterId of uniqueClusterIds) { const clusterName = clusterNames[clusterId] || `Cluster ${clusterId + 1}`; - const { id: childFolderId, created } = await getOrCreateFolder(existingFoldersMap, clusterName); + const { id: childFolderId, created } = await getOrCreateFolder( + existingFoldersMap, + clusterName, + rootParentFolderId, + ); folderMap[clusterId] = childFolderId; if (created) { createdFolderIds.push(childFolderId); @@ -64,7 +79,11 @@ export async function initializeClusterNotebooks( } if (assignments.includes(-1)) { - const { id: noiseFolderId, created } = await getOrCreateFolder(existingFoldersMap, 'Uncategorized'); + const { id: noiseFolderId, created } = await getOrCreateFolder( + existingFoldersMap, + 'Uncategorized', + rootParentFolderId, + ); uncategorizedFolderId = noiseFolderId; if (created) { createdFolderIds.push(noiseFolderId); diff --git a/src/commands/registerCommands.ts b/src/commands/registerCommands.ts new file mode 100644 index 0000000..ed787d8 --- /dev/null +++ b/src/commands/registerCommands.ts @@ -0,0 +1,62 @@ +import joplin from 'api'; +import { MenuItemLocation, ToolbarButtonLocation } from 'api/types'; +import { log } from '../utils/logger'; +import { runNativeUndo, runNativeCleanup, OperationState } from '../settings/registerSettings'; + +export async function registerPluginCommands(operationState: OperationState, panelHandle: string): Promise { + await joplin.commands.register({ + name: 'aiCategorise.undoLastCategorization', + label: 'AI Categorise: Undo Last Categorization', + iconName: 'fas fa-undo', + execute: async () => { + log('Menu: triggering undoCategorizationChanges'); + await runNativeUndo('Menu', operationState); + }, + }); + + await joplin.commands.register({ + name: 'aiCategorise.cleanUpEmptyNotebooks', + label: 'AI Categorise: Clean Up Empty Notebooks', + iconName: 'fas fa-broom', + execute: async () => { + log('Menu: triggering cleanUpEmptyNotebooks'); + await runNativeCleanup('Menu', operationState); + }, + }); + + await joplin.commands.register({ + name: 'aiCategorise.togglePanel', + label: 'AI Categorise: Toggle Panel', + iconName: 'fas fa-brain', + execute: async () => { + const visible = await joplin.views.panels.visible(panelHandle); + await joplin.views.panels.show(panelHandle, !visible); + }, + }); + + await joplin.views.menuItems.create( + 'aiCategorise.undoMenuItem', + 'aiCategorise.undoLastCategorization', + MenuItemLocation.Tools, + ); + + await joplin.views.menuItems.create( + 'aiCategorise.cleanUpMenuItem', + 'aiCategorise.cleanUpEmptyNotebooks', + MenuItemLocation.Tools, + ); + + await joplin.views.menuItems.create( + 'aiCategorise.togglePanelMenuItem', + 'aiCategorise.togglePanel', + MenuItemLocation.View, + ); + + await joplin.views.toolbarButtons.create( + 'aiCategorise.togglePanelToolbar', + 'aiCategorise.togglePanel', + ToolbarButtonLocation.NoteToolbar, + ); + + log('Commands and menu items registered'); +} diff --git a/src/index.ts b/src/index.ts index 7c4f940..a4ea2b3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,175 +1,25 @@ import joplin from 'api'; -import { MenuItemLocation, ToolbarButtonLocation, SettingItemType as SettingType } from 'api/types'; -import { runPipeline } from './pipeline/runPipeline'; -import { PanelMessage, WebviewMessage } from './types/panel'; import { log } from './utils/logger'; -import { applyCategorizationChanges, undoCategorizationChanges, cleanUpEmptyNotebooks } from './commands/applyChanges'; +import { registerPluginSettings, OperationState } from './settings/registerSettings'; +import { registerPluginCommands } from './commands/registerCommands'; +import { setupPanel } from './panel/setupPanel'; joplin.plugins.register({ onStart: async function () { log('Plugin started'); - // Register setting section - await joplin.settings.registerSection('aiCategorization', { - label: 'AI Categorization', - iconName: 'fas fa-brain', - }); + // Shared operation lock across webview IPC and native triggers + const operationState: OperationState = { inProgress: false }; - // Register setting items - await joplin.settings.registerSettings({ - 'categorization.changeLog': { - value: '', - type: SettingType.String, - section: 'aiCategorization', - public: false, - label: 'Change Log', - description: 'Stores previous states of moved and tagged notes for undo operations.', - }, - }); + // 1. Register plugin settings section & items + await registerPluginSettings(operationState); - const installDir = await joplin.plugins.installationDir(); + // 2. Setup webview side panel & IPC onMessage loop + const panelHandle = await setupPanel(operationState); - // Panel starts hidden; user opens via toolbar button or View menu - const panel = await joplin.views.panels.create('aiCategorise.panel'); - await joplin.views.panels.setHtml(panel, '
'); - await joplin.views.panels.addScript(panel, './webview/panel.css'); - await joplin.views.panels.addScript(panel, './webview/panel.js'); - await joplin.views.panels.show(panel, false); + // 3. Register Joplin commands, menu items, & toolbar buttons + await registerPluginCommands(operationState, panelHandle); - // Pipeline state shared between the onMessage handler and pipeline callbacks. - // The webview polls this state via { type: 'poll' } messages. - let panelState: PanelMessage | { type: 'idle' } = { type: 'idle' }; - let operationInProgress = false; - - await joplin.views.panels.onMessage(panel, async (msg: WebviewMessage) => { - switch (msg.type) { - case 'run': - panelState = { type: 'status', text: 'Starting pipeline...' }; - log('Panel: starting pipeline'); - - // Fire-and-forget — pipeline updates panelState via callbacks - runPipeline(installDir, { - onStatus: (text) => { - panelState = { type: 'status', text }; - }, - onProgress: (current, total, cached, skipped) => { - panelState = { type: 'progress', current, total, cached, skipped }; - }, - onComplete: (strategies, notes) => { - panelState = { type: 'results', strategies, notes }; - }, - onError: (message) => { - panelState = { type: 'error', message }; - }, - }); - - return panelState; - - case 'poll': - return panelState; - - case 'openNote': - if (msg.noteId) { - await joplin.commands.execute('openNote', msg.noteId); - } - return; - - case 'getSettings': - return { - 'categorization.metric': 'cosine', - 'categorization.parentNotebook': '', - 'categorization.changeLog': await joplin.settings.value('categorization.changeLog'), - }; - - case 'updateSetting': - await joplin.settings.setValue(msg.key, msg.value); - return { success: true }; - - case 'apply': - if (operationInProgress) { - return { type: 'apply_error', message: 'Another operation is already in progress.' }; - } - operationInProgress = true; - panelState = { type: 'apply_status', text: 'Initializing application of categorization...' }; - applyCategorizationChanges( - msg.options, - msg.notes, - msg.assignments, - msg.clusterNames, - msg.clusterTags, - (state) => { - panelState = state; - }, - ) - .catch((err) => { - log('Error in apply background task: ' + err); - panelState = { type: 'apply_error', message: err.message || String(err) }; - }) - .finally(() => { - operationInProgress = false; - }); - return panelState; - - case 'undo': - if (operationInProgress) { - return { type: 'undo_error', message: 'Another operation is already in progress.' }; - } - operationInProgress = true; - panelState = { type: 'undo_status', text: 'Initializing undo...' }; - undoCategorizationChanges((state) => { - panelState = state; - }) - .catch((err) => { - log('Error in undo background task: ' + err); - panelState = { type: 'undo_error', message: err.message || String(err) }; - }) - .finally(() => { - operationInProgress = false; - }); - return panelState; - - case 'cleanUpEmptyNotebooks': - if (operationInProgress) { - return { type: 'cleanup_error', message: 'Another operation is already in progress.' }; - } - operationInProgress = true; - panelState = { type: 'cleanup_status', text: 'Checking empty notebooks...' }; - cleanUpEmptyNotebooks((state) => { - panelState = state; - }) - .catch((err) => { - log('Error in cleanup background task: ' + err); - panelState = { type: 'cleanup_error', message: err.message || String(err) }; - }) - .finally(() => { - operationInProgress = false; - }); - return panelState; - } - }); - - await joplin.commands.register({ - name: 'aiCategorise.togglePanel', - label: 'AI Categorise: Toggle Panel', - iconName: 'fas fa-brain', - execute: async () => { - const visible = await joplin.views.panels.visible(panel); - await joplin.views.panels.show(panel, !visible); - }, - }); - - await joplin.views.menuItems.create( - 'aiCategorise.togglePanelMenuItem', - 'aiCategorise.togglePanel', - MenuItemLocation.View, - ); - - await joplin.views.toolbarButtons.create( - 'aiCategorise.togglePanelToolbar', - 'aiCategorise.togglePanel', - ToolbarButtonLocation.NoteToolbar, - ); - - log('Panel registered'); + log('Plugin setup complete'); }, }); diff --git a/src/panel/setupPanel.ts b/src/panel/setupPanel.ts new file mode 100644 index 0000000..901823d --- /dev/null +++ b/src/panel/setupPanel.ts @@ -0,0 +1,157 @@ +import joplin from 'api'; +import { runPipeline } from '../pipeline/runPipeline'; +import { PanelMessage, WebviewMessage, PanelNote } from '../types/panel'; +import { BenchmarkResult } from '../types/cluster'; +import { log } from '../utils/logger'; +import { applyCategorizationChanges, undoCategorizationChanges, cleanUpEmptyNotebooks } from '../commands/applyChanges'; +import { OperationState } from '../settings/registerSettings'; + +export async function setupPanel(operationState: OperationState): Promise { + const installDir = await joplin.plugins.installationDir(); + + const panel = await joplin.views.panels.create('aiCategorise.panel'); + await joplin.views.panels.setHtml(panel, '
'); + await joplin.views.panels.addScript(panel, './webview/panel.css'); + await joplin.views.panels.addScript(panel, './webview/panel.js'); + await joplin.views.panels.show(panel, false); + + let panelState: PanelMessage | { type: 'idle' } = { type: 'idle' }; + let lastResultsState: { + strategies: BenchmarkResult[]; + notes: PanelNote[]; + selectedStrategyIndex: number; + } | null = null; + + await joplin.views.panels.onMessage(panel, async (msg: WebviewMessage) => { + switch (msg.type) { + case 'run': + panelState = { type: 'status', text: 'Starting pipeline...' }; + log('Panel: starting pipeline'); + + runPipeline(installDir, { + onStatus: (text) => { + panelState = { type: 'status', text }; + }, + onProgress: (current, total, cached, skipped) => { + panelState = { type: 'progress', current, total, cached, skipped }; + }, + onComplete: (strategies, notes) => { + lastResultsState = { strategies, notes, selectedStrategyIndex: 0 }; + panelState = { type: 'results', strategies, notes }; + }, + onError: (message) => { + panelState = { type: 'error', message }; + }, + }); + + return panelState; + + case 'poll': + return panelState; + + case 'getInitialState': + if (panelState.type === 'status' || panelState.type === 'progress') { + return panelState; + } + if (lastResultsState) { + return { + type: 'results', + strategies: lastResultsState.strategies, + notes: lastResultsState.notes, + selectedStrategyIndex: lastResultsState.selectedStrategyIndex, + }; + } + return panelState; + + case 'syncState': + lastResultsState = { + strategies: msg.strategies, + notes: msg.notes, + selectedStrategyIndex: msg.selectedStrategyIndex, + }; + return { success: true }; + + case 'openNote': + if (msg.noteId) { + await joplin.commands.execute('openNote', msg.noteId); + } + return; + + case 'getSettings': + return { + 'categorization.metric': await joplin.settings.value('categorization.metric'), + 'categorization.parentNotebook': await joplin.settings.value('categorization.parentNotebook'), + 'categorization.seed': await joplin.settings.value('categorization.seed'), + 'categorization.changeLog': await joplin.settings.value('categorization.changeLog'), + }; + + case 'updateSetting': + await joplin.settings.setValue(msg.key, msg.value); + return { success: true }; + + case 'apply': + if (operationState.inProgress) { + return { type: 'apply_error', message: 'Another operation is already in progress.' }; + } + operationState.inProgress = true; + panelState = { type: 'apply_status', text: 'Initializing application of categorization...' }; + applyCategorizationChanges( + msg.options, + msg.notes, + msg.assignments, + msg.clusterNames, + msg.clusterTags, + (state) => { + panelState = state; + }, + ) + .catch((err) => { + log('Error in apply background task: ' + err); + panelState = { type: 'apply_error', message: err.message || String(err) }; + }) + .finally(() => { + operationState.inProgress = false; + }); + return panelState; + + case 'undo': + if (operationState.inProgress) { + return { type: 'undo_error', message: 'Another operation is already in progress.' }; + } + operationState.inProgress = true; + panelState = { type: 'undo_status', text: 'Initializing undo...' }; + undoCategorizationChanges((state) => { + panelState = state; + }) + .catch((err) => { + log('Error in undo background task: ' + err); + panelState = { type: 'undo_error', message: err.message || String(err) }; + }) + .finally(() => { + operationState.inProgress = false; + }); + return panelState; + + case 'cleanUpEmptyNotebooks': + if (operationState.inProgress) { + return { type: 'cleanup_error', message: 'Another operation is already in progress.' }; + } + operationState.inProgress = true; + panelState = { type: 'cleanup_status', text: 'Checking empty notebooks...' }; + cleanUpEmptyNotebooks((state) => { + panelState = state; + }) + .catch((err) => { + log('Error in cleanup background task: ' + err); + panelState = { type: 'cleanup_error', message: err.message || String(err) }; + }) + .finally(() => { + operationState.inProgress = false; + }); + return panelState; + } + }); + + log('Panel setup complete'); + return panel; +} diff --git a/src/settings/registerSettings.ts b/src/settings/registerSettings.ts new file mode 100644 index 0000000..319ac7a --- /dev/null +++ b/src/settings/registerSettings.ts @@ -0,0 +1,162 @@ +import joplin from 'api'; +import { SettingItemType as SettingType } from 'api/types'; +import { log } from '../utils/logger'; +import { undoCategorizationChanges, cleanUpEmptyNotebooks } from '../commands/applyChanges'; + +export interface OperationState { + inProgress: boolean; +} + +const OP_IN_PROGRESS_MSG = 'An operation is already in progress. Please wait for it to complete.'; + +export async function runNativeUndo(source: string, operationState: OperationState): Promise { + if (operationState.inProgress) { + await joplin.views.dialogs.showMessageBox(OP_IN_PROGRESS_MSG); + return; + } + operationState.inProgress = true; + try { + let lastMessage = ''; + await undoCategorizationChanges((state) => { + log(`Native ${source} Undo: ${'text' in state ? state.text : state.type}`); + if (state.type === 'undo_complete') { + lastMessage = 'Reverted categorization changes successfully!'; + } else if (state.type === 'undo_error') { + lastMessage = `Undo Error: ${state.message}`; + } + }); + if (lastMessage) { + await joplin.views.dialogs.showMessageBox(lastMessage); + } + } catch (err) { + await joplin.views.dialogs.showMessageBox(`Undo failed: ${err instanceof Error ? err.message : String(err)}`); + } finally { + operationState.inProgress = false; + } +} + +export async function runNativeCleanup(source: string, operationState: OperationState): Promise { + if (operationState.inProgress) { + await joplin.views.dialogs.showMessageBox(OP_IN_PROGRESS_MSG); + return; + } + operationState.inProgress = true; + try { + let lastMessage = ''; + await cleanUpEmptyNotebooks((state) => { + log(`Native ${source} Cleanup: ${'text' in state ? state.text : state.type}`); + if (state.type === 'cleanup_complete') { + lastMessage = state.message; + } else if (state.type === 'cleanup_error') { + lastMessage = `Cleanup Error: ${state.message}`; + } + }); + if (lastMessage) { + await joplin.views.dialogs.showMessageBox(lastMessage); + } + } catch (err) { + await joplin.views.dialogs.showMessageBox( + `Cleanup failed: ${err instanceof Error ? err.message : String(err)}`, + ); + } finally { + operationState.inProgress = false; + } +} + +export async function registerPluginSettings(operationState: OperationState): Promise { + try { + await joplin.settings.registerSection('aiCategorization', { + label: 'AI Categorization', + iconName: 'fas fa-brain', + }); + + await joplin.settings.registerSettings({ + 'categorization.metric': { + value: 'cosine', + type: SettingType.String, + isEnum: true, + options: { + cosine: 'Cosine Similarity (Recommended)', + euclidean: 'Euclidean Distance', + }, + section: 'aiCategorization', + public: true, + label: 'Distance Metric', + description: 'Distance metric used for note embedding comparisons and clustering.', + }, + 'categorization.parentNotebook': { + value: '', + type: SettingType.String, + section: 'aiCategorization', + public: true, + label: 'Default Target Notebook', + description: + 'Default parent notebook where newly categorized sub-notebooks will be created (leave empty for root).', + }, + 'categorization.seed': { + value: 42, + type: SettingType.Int, + section: 'aiCategorization', + public: true, + label: 'Random Seed', + description: 'Random seed for reproducible UMAP projections and K-Means clustering.', + }, + 'categorization.changeLog': { + value: '', + type: SettingType.String, + section: 'aiCategorization', + public: false, + label: 'Change Log', + description: 'Stores previous states of moved and tagged notes for undo operations.', + }, + 'categorization.changeLogSummary': { + value: 'No categorization has been applied yet.', + type: SettingType.String, + section: 'aiCategorization', + public: true, + label: 'Last Categorization Summary', + description: 'Summary of the last applied categorization.', + }, + 'categorization.undoAction': { + value: false, + type: SettingType.Bool, + section: 'aiCategorization', + public: true, + label: 'Undo Last Categorization', + description: + 'Check this box and click Apply/OK to revert note movements and tags from the previous run.', + }, + 'categorization.cleanUpAction': { + value: false, + type: SettingType.Bool, + section: 'aiCategorization', + public: true, + label: 'Clean Up Empty Notebooks', + description: + 'Check this box and click Apply/OK to check for and remove empty notebooks leftover after note moves.', + }, + }); + + // Handle native options checkbox triggers + await joplin.settings.onChange(async (event: { keys: string[] }) => { + if (event.keys.includes('categorization.undoAction')) { + const val = await joplin.settings.value('categorization.undoAction'); + if (val) { + await joplin.settings.setValue('categorization.undoAction', false); + log('Native Settings: triggering undoCategorizationChanges'); + await runNativeUndo('Settings', operationState); + } + } + if (event.keys.includes('categorization.cleanUpAction')) { + const val = await joplin.settings.value('categorization.cleanUpAction'); + if (val) { + await joplin.settings.setValue('categorization.cleanUpAction', false); + log('Native Settings: triggering cleanUpEmptyNotebooks'); + await runNativeCleanup('Settings', operationState); + } + } + }); + } catch (err) { + log('Error registering settings: ' + err); + } +} diff --git a/src/types/panel.ts b/src/types/panel.ts index a3deb9c..81e3f42 100644 --- a/src/types/panel.ts +++ b/src/types/panel.ts @@ -32,7 +32,7 @@ export interface ApplyMessage { export type PanelMessage = | { type: 'status'; text: string } | { type: 'progress'; current: number; total: number; cached: number; skipped: number } - | { type: 'results'; strategies: BenchmarkResult[]; notes: PanelNote[] } + | { type: 'results'; strategies: BenchmarkResult[]; notes: PanelNote[]; selectedStrategyIndex?: number } | { type: 'error'; message: string } | { type: 'apply_status'; text: string } | { type: 'apply_progress'; current: number; total: number } @@ -50,6 +50,8 @@ export type PanelMessage = export type WebviewMessage = | { type: 'run' } | { type: 'poll' } + | { type: 'getInitialState' } + | { type: 'syncState'; strategies: BenchmarkResult[]; notes: PanelNote[]; selectedStrategyIndex: number } | { type: 'openNote'; noteId: string } | { type: 'getSettings' } | { type: 'updateSetting'; key: string; value: string } diff --git a/src/webview/components/Navigation.tsx b/src/webview/components/Navigation.tsx deleted file mode 100644 index 1df0ddd..0000000 --- a/src/webview/components/Navigation.tsx +++ /dev/null @@ -1,37 +0,0 @@ -import * as React from 'react'; -import { useAppState, ViewType } from '../context/AppStateContext'; - -export const Navigation: React.FC = () => { - const { activeView, setView, strategies } = useAppState(); - - const hasResults = strategies && strategies.length > 0; - - const handleTabClick = (view: ViewType) => { - setView(view); - }; - - return ( -
- - {hasResults && ( - - )} - -
- ); -}; diff --git a/src/webview/context/AppStateContext.tsx b/src/webview/context/AppStateContext.tsx index b3c57e2..ba41c17 100644 --- a/src/webview/context/AppStateContext.tsx +++ b/src/webview/context/AppStateContext.tsx @@ -6,7 +6,7 @@ import { usePipelineState } from './usePipelineState'; const POLL_INTERVAL_MS = 500; -export type ViewType = 'idle' | 'dashboard' | 'history' | 'settings'; +export type ViewType = 'idle' | 'dashboard'; interface AppStateContextType { isRunning: boolean; @@ -153,7 +153,8 @@ export const AppStateProvider: React.FC<{ children: React.ReactNode }> = ({ chil (s: BenchmarkResult) => !s.strategyName.startsWith('kmeans') && !s.strategyName.startsWith('kmedoids'), ); - setSelectedStrategyIndex(nonTestingIdx !== -1 ? nonTestingIdx : 0); + const fallbackIdx = nonTestingIdx !== -1 ? nonTestingIdx : 0; + setSelectedStrategyIndex(msg.selectedStrategyIndex ?? fallbackIdx); setError(null); setActiveView('dashboard'); break; @@ -267,16 +268,51 @@ export const AppStateProvider: React.FC<{ children: React.ReactNode }> = ({ chil const startPolling = React.useCallback(() => { stopPolling(); pollIntervalRef.current = setInterval(async () => { - const state = await webviewApi.postMessage({ type: 'poll' }); - if (state) { - handlePollResponse(state); + if (typeof webviewApi === 'undefined') return; + try { + const state = await webviewApi.postMessage({ type: 'poll' }); + if (state) { + handlePollResponse(state); + } + } catch (err) { + console.error('Polling error:', err); } }, POLL_INTERVAL_MS); }, [stopPolling, handlePollResponse]); React.useEffect(() => { fetchSettings(); - }, [fetchSettings]); + if (typeof webviewApi !== 'undefined') { + webviewApi + .postMessage({ type: 'getInitialState' }) + .then((initialState) => { + if (initialState) { + handlePollResponse(initialState); + if (initialState.type === 'status' || initialState.type === 'progress') { + startPolling(); + } + } + }) + .catch((err) => { + console.error('getInitialState error:', err); + }); + } + }, [fetchSettings, handlePollResponse, startPolling]); + + React.useEffect(() => { + if (typeof webviewApi !== 'undefined' && strategies && strategies.length > 0) { + webviewApi + .postMessage({ + type: 'syncState', + strategies, + notes, + selectedStrategyIndex, + }) + .catch((err) => { + console.error('syncState error:', err); + }); + } + }, [strategies, notes, selectedStrategyIndex]); React.useEffect(() => { return () => { @@ -284,10 +320,13 @@ export const AppStateProvider: React.FC<{ children: React.ReactNode }> = ({ chil }; }, [stopPolling]); - const handleApplyChanges = async (options: ApplyOptions) => { - const currentStrategy = strategies[selectedStrategyIndex]; - await applyChanges(options, notes, currentStrategy); - }; + const handleApplyChanges = React.useCallback( + async (options: ApplyOptions) => { + const currentStrategy = strategies[selectedStrategyIndex]; + await applyChanges(options, notes, currentStrategy); + }, + [strategies, selectedStrategyIndex, notes, applyChanges], + ); return ( void) { setCleanupError(null); }, []); - const applyChanges = async ( - options: ApplyOptions, - notes: PanelNote[], - currentStrategy: BenchmarkResult | undefined, - ) => { - if (!currentStrategy) { - setApplyError('No active strategy selected.'); - return; - } + const applyChanges = React.useCallback( + async (options: ApplyOptions, notes: PanelNote[], currentStrategy: BenchmarkResult | undefined) => { + if (!currentStrategy) { + setApplyError('No active strategy selected.'); + return; + } - setIsApplying(true); - setApplyProgress({ current: 0, total: notes.length }); - setApplyError(null); - setApplySuccess(false); - setUndoSuccess(false); - setUndoError(null); - setCleanupSuccess(null); - setCleanupError(null); + setIsApplying(true); + setApplyProgress({ current: 0, total: notes.length }); + setApplyError(null); + setApplySuccess(false); + setUndoSuccess(false); + setUndoError(null); + setCleanupSuccess(null); + setCleanupError(null); - try { - await webviewApi.postMessage({ - type: 'apply', - options, - notes, - assignments: currentStrategy.assignments, - clusterNames: currentStrategy.clusterNames || {}, - clusterTags: currentStrategy.tags || {}, - }); - startPolling(); - } catch (err) { - setApplyError('Failed to apply changes: ' + String(err)); - setIsApplying(false); - } - }; + try { + if (typeof webviewApi === 'undefined') { + setApplyError('Joplin API not available'); + setIsApplying(false); + return; + } + await webviewApi.postMessage({ + type: 'apply', + options, + notes, + assignments: currentStrategy.assignments, + clusterNames: currentStrategy.clusterNames || {}, + clusterTags: currentStrategy.tags || {}, + }); + startPolling(); + } catch (err) { + setApplyError('Failed to apply changes: ' + String(err)); + setIsApplying(false); + } + }, + [startPolling], + ); - const undoChanges = async () => { + const undoChanges = React.useCallback(async () => { setIsUndoing(true); setUndoProgress({ current: 0, total: 0 }); setUndoError(null); @@ -71,15 +75,20 @@ export function useApplyState(startPolling: () => void) { setCleanupError(null); try { + if (typeof webviewApi === 'undefined') { + setUndoError('Joplin API not available'); + setIsUndoing(false); + return; + } await webviewApi.postMessage({ type: 'undo' }); startPolling(); } catch (err) { setUndoError('Failed to start undo operation: ' + String(err)); setIsUndoing(false); } - }; + }, [startPolling]); - const cleanUpNotebooks = async () => { + const cleanUpNotebooks = React.useCallback(async () => { setIsCleaningUp(true); setCleanupError(null); setCleanupSuccess(null); @@ -89,13 +98,18 @@ export function useApplyState(startPolling: () => void) { setUndoError(null); try { + if (typeof webviewApi === 'undefined') { + setCleanupError('Joplin API not available'); + setIsCleaningUp(false); + return; + } await webviewApi.postMessage({ type: 'cleanUpEmptyNotebooks' }); startPolling(); } catch (err) { setCleanupError('Failed to start cleanup: ' + String(err)); setIsCleaningUp(false); } - }; + }, [startPolling]); return { isApplying, diff --git a/src/webview/context/usePipelineState.ts b/src/webview/context/usePipelineState.ts index 86616b8..7d7020e 100644 --- a/src/webview/context/usePipelineState.ts +++ b/src/webview/context/usePipelineState.ts @@ -30,6 +30,11 @@ export function usePipelineState(startPolling: () => void, resetApplyState: () = resetApplyState(); try { + if (typeof webviewApi === 'undefined') { + setError('Joplin plugin API not available'); + setIsRunning(false); + return; + } await webviewApi.postMessage({ type: 'run' }); } catch (err) { setError('Failed to start pipeline: ' + String(err)); @@ -49,6 +54,7 @@ export function usePipelineState(startPolling: () => void, resetApplyState: () = const updateClusterName = React.useCallback( (clusterId: number, newName: string) => { + resetApplyState(); setStrategies((prev) => { const next = [...prev]; if (next[selectedStrategyIndex]) { @@ -61,11 +67,12 @@ export function usePipelineState(startPolling: () => void, resetApplyState: () = return next; }); }, - [selectedStrategyIndex], + [selectedStrategyIndex, resetApplyState], ); const moveNoteToCluster = React.useCallback( (noteIndex: number, targetClusterId: number) => { + resetApplyState(); setStrategies((prev) => { const next = [...prev]; if (next[selectedStrategyIndex]) { @@ -82,7 +89,7 @@ export function usePipelineState(startPolling: () => void, resetApplyState: () = return next; }); }, - [selectedStrategyIndex], + [selectedStrategyIndex, resetApplyState], ); const addCluster = React.useCallback( @@ -100,6 +107,7 @@ export function usePipelineState(startPolling: () => void, resetApplyState: () = return false; } + resetApplyState(); setStrategies((prev) => { const next = [...prev]; const strat = next[selectedStrategyIndex]; @@ -125,7 +133,7 @@ export function usePipelineState(startPolling: () => void, resetApplyState: () = return true; }, - [strategies, selectedStrategyIndex], + [strategies, selectedStrategyIndex, resetApplyState], ); return { diff --git a/src/webview/context/useSettingsState.ts b/src/webview/context/useSettingsState.ts index 0f3fdb0..ad220a0 100644 --- a/src/webview/context/useSettingsState.ts +++ b/src/webview/context/useSettingsState.ts @@ -3,6 +3,7 @@ import * as React from 'react'; interface SettingsResponse { 'categorization.metric': string; 'categorization.parentNotebook': string; + 'categorization.seed': number; 'categorization.changeLog': string; } @@ -10,17 +11,20 @@ export function useSettingsState() { const [settings, setSettings] = React.useState({ metric: 'cosine', parentNotebook: '', + seed: 42, changeLog: '', }); const fetchSettings = React.useCallback(async () => { try { + if (typeof webviewApi === 'undefined') return; const res = await webviewApi.postMessage({ type: 'getSettings' }); if (res) { const data = res as unknown as SettingsResponse; setSettings({ metric: data['categorization.metric'] || 'cosine', parentNotebook: data['categorization.parentNotebook'] || '', + seed: data['categorization.seed'] ?? 42, changeLog: data['categorization.changeLog'] || '', }); } @@ -31,6 +35,7 @@ export function useSettingsState() { const updateSetting = React.useCallback(async (key: string, value: string) => { try { + if (typeof webviewApi === 'undefined') return; await webviewApi.postMessage({ type: 'updateSetting', key, diff --git a/src/webview/pages/DashboardPage.tsx b/src/webview/pages/DashboardPage.tsx index 2896156..a9160f4 100644 --- a/src/webview/pages/DashboardPage.tsx +++ b/src/webview/pages/DashboardPage.tsx @@ -22,6 +22,7 @@ export const DashboardPage: React.FC = () => { applyChanges, isUndoing, isCleaningUp, + settings, } = useAppState(); const selectedStrategy = strategies[selectedStrategyIndex]; @@ -30,30 +31,34 @@ export const DashboardPage: React.FC = () => { const [newClusterName, setNewClusterName] = React.useState(''); const [duplicateError, setDuplicateError] = React.useState(false); - const clusters: { [key: number]: number[] } = {}; - const noise: number[] = []; - - if (selectedStrategy) { - const clusterNames = selectedStrategy.clusterNames || {}; - Object.keys(clusterNames).forEach((clusterId) => { - clusters[Number(clusterId)] = []; - }); - - selectedStrategy.assignments.forEach((clusterId, noteIndex) => { - if (clusterId === -1) { - noise.push(noteIndex); - } else { - if (!clusters[clusterId]) { - clusters[clusterId] = []; + const { clusters, noise, sortedClusterIds } = React.useMemo(() => { + const clusters: { [key: number]: number[] } = {}; + const noise: number[] = []; + + if (selectedStrategy) { + const clusterNames = selectedStrategy.clusterNames || {}; + Object.keys(clusterNames).forEach((clusterId) => { + clusters[Number(clusterId)] = []; + }); + + selectedStrategy.assignments.forEach((clusterId, noteIndex) => { + if (clusterId === -1) { + noise.push(noteIndex); + } else { + if (!clusters[clusterId]) { + clusters[clusterId] = []; + } + clusters[clusterId].push(noteIndex); } - clusters[clusterId].push(noteIndex); - } - }); - } + }); + } + + const sortedClusterIds = Object.keys(clusters) + .map(Number) + .sort((a, b) => clusters[b].length - clusters[a].length); - const sortedClusterIds = Object.keys(clusters) - .map(Number) - .sort((a, b) => clusters[b].length - clusters[a].length); + return { clusters, noise, sortedClusterIds }; + }, [selectedStrategy]); const handleAddClusterSubmit = (e?: React.FormEvent) => { if (e) e.preventDefault(); @@ -71,9 +76,12 @@ export const DashboardPage: React.FC = () => { }; const handleApply = () => { + if (applySuccess || isApplying || isUndoing || isCleaningUp) { + return; + } applyChanges({ method: 'both', - parentNotebookName: '', + parentNotebookName: settings.parentNotebook || '', }); }; @@ -173,9 +181,13 @@ export const DashboardPage: React.FC = () => { @@ -186,7 +198,10 @@ export const DashboardPage: React.FC = () => { )} {applySuccess && ( -
Categorization applied successfully!
+
+ Categorization applied successfully! To undo or clean up, go to Tools → Options → AI + Categorization. +
)} {applyError &&
Error: {applyError}
} diff --git a/src/webview/pages/SettingsPage.tsx b/src/webview/pages/SettingsPage.tsx index d554109..54cfa47 100644 --- a/src/webview/pages/SettingsPage.tsx +++ b/src/webview/pages/SettingsPage.tsx @@ -1,25 +1,33 @@ import * as React from 'react'; +import { useAppState } from '../context/AppStateContext'; export const SettingsPage: React.FC = () => { + const { settings } = useAppState(); + return ( -
-
Settings
+
+
Plugin Settings
- Model settings and clustering parameters will be configurable here in a future version. + Settings are managed in Joplin's native Options window. Go to{' '} + Tools → Options → AI Categorization to configure the plugin.
-
-
Default Configuration:
+ +
+
Active Configuration:
- • Model: Xenova/all-MiniLM-L6-v2 (384-dim) + • Distance Metric: {settings.metric || 'Cosine Similarity'}
- • Metric: Cosine Similarity + • Target Notebook: {settings.parentNotebook || '(Root Notebooks)'}
- • Limit: 200 Tokens/Chunk + • Embedding Model: all-MiniLM-L6-v2 (384-dim)
- • Strategies: K-Means, K-Medoids, HDBSCAN + • Clustering Strategies: Auto K-Means, Auto K-Medoids, HDBSCAN
diff --git a/src/webview/panel.css b/src/webview/panel.css index 96b2500..b4efb35 100644 --- a/src/webview/panel.css +++ b/src/webview/panel.css @@ -40,42 +40,6 @@ body { flex-direction: column; } -/* NAVIGATION TABS */ - -.panel-navigation { - display: flex; - gap: 0; - background: var(--joplin-background-color); - border-bottom: 1px solid var(--joplin-divider-color); - padding: 0 16px; -} - -.nav-tab { - position: relative; - padding: 10px 14px; - background: transparent; - border: none; - border-bottom: 2px solid transparent; - color: var(--joplin-color); - opacity: 0.5; - font-family: inherit; - font-size: 0.85em; - font-weight: 500; - cursor: pointer; - transition: opacity 150ms ease, border-color 150ms ease; - letter-spacing: 0.01em; -} - -.nav-tab:hover { - opacity: 0.8; -} - -.nav-tab.active { - opacity: 1; - font-weight: 600; - border-bottom-color: var(--joplin-color); -} - /* HEADER */ .panel-header { @@ -296,7 +260,7 @@ body { display: inline-block; padding: 3px 10px; border-radius: 100px; - font-size: 0.72em; + font-size: 0.8em; font-weight: 500; background: color-mix(in srgb, var(--joplin-color) 4%, var(--joplin-background-color)); border: 1px solid var(--joplin-divider-color); @@ -386,7 +350,7 @@ body { } .cluster-title { - font-size: 0.88em; + font-size: 0.95em; font-weight: 600; white-space: nowrap; overflow: hidden; @@ -416,7 +380,7 @@ body { } .cluster-title-input { - font-size: 0.88em; + font-size: 0.95em; font-weight: 600; font-family: inherit; padding: 2px 6px; @@ -435,7 +399,7 @@ body { flex-wrap: wrap; gap: 4px; margin-left: 0; - max-height: 20px; + max-height: 24px; overflow: hidden; width: 100%; } @@ -446,12 +410,12 @@ body { } .cluster-tag { - font-size: 0.7em; + font-size: 0.82em; font-weight: 500; color: var(--joplin-color); - opacity: 0.5; + opacity: 0.65; background: color-mix(in srgb, var(--joplin-color) 4%, var(--joplin-background-color)); - padding: 1px 7px; + padding: 2px 8px; border: 1px solid color-mix(in srgb, var(--joplin-divider-color) 60%, transparent); border-radius: 100px; white-space: nowrap; @@ -460,8 +424,8 @@ body { /* COUNT & CHEVRON */ .cluster-count { - font-size: 0.75em; - opacity: 0.4; + font-size: 0.82em; + opacity: 0.5; white-space: nowrap; flex-shrink: 0; margin-left: 8px; @@ -504,9 +468,9 @@ body { display: flex; align-items: center; gap: 8px; - padding: 7px 14px 7px 28px; + padding: 8px 14px 8px 28px; cursor: pointer; - font-size: 0.82em; + font-size: 0.9em; transition: background 100ms ease; } diff --git a/src/webview/panel.tsx b/src/webview/panel.tsx index 36f14e5..f08b14a 100644 --- a/src/webview/panel.tsx +++ b/src/webview/panel.tsx @@ -1,27 +1,17 @@ import * as React from 'react'; import { createRoot } from 'react-dom/client'; import { AppStateProvider, useAppState } from './context/AppStateContext'; -import { Navigation } from './components/Navigation'; import { DashboardPage } from './pages/DashboardPage'; import { EmptyStatePage } from './pages/EmptyStatePage'; -import { HistoryPage } from './pages/HistoryPage'; -import { SettingsPage } from './pages/SettingsPage'; const AppContent: React.FC = () => { const { activeView, error } = useAppState(); return (
- - {error &&
Error: {error}
} -
- {activeView === 'idle' && } - {activeView === 'dashboard' && } - {activeView === 'history' && } - {activeView === 'settings' && } -
+
{activeView === 'idle' ? : }
); }; From 367ec3aedff8e0be5f44c1f2619239b88354dbbf Mon Sep 17 00:00:00 2001 From: Harsh16gupta Date: Mon, 3 Aug 2026 02:50:51 +0530 Subject: [PATCH 2/3] feat(clustering): improve auto-K selection algorithm and simplify settings --- src/panel/setupPanel.ts | 2 - src/pipeline/clustering/autoK.ts | 159 +++++++++++++++++ src/pipeline/clustering/benchmark.ts | 46 ++++- src/pipeline/pipelineConfig.ts | 46 +++-- src/pipeline/runPipeline.ts | 13 +- src/pipeline/vectorAggregator.ts | 33 ++++ src/settings/registerSettings.ts | 34 +--- src/types/cluster.ts | 8 +- src/types/panel.ts | 7 +- src/webview/context/AppStateContext.tsx | 21 +-- src/webview/context/useApplyState.ts | 6 +- src/webview/context/useSettingsState.ts | 6 - src/webview/pages/SettingsPage.tsx | 11 +- src/webview/panel.tsx | 4 +- test/pipeline/clustering/autoK.test.ts | 228 ++++++++++++++++++++++++ test/pipeline/vectorAggregator.test.ts | 53 ++++++ 16 files changed, 587 insertions(+), 90 deletions(-) create mode 100644 src/pipeline/clustering/autoK.ts create mode 100644 test/pipeline/clustering/autoK.test.ts diff --git a/src/panel/setupPanel.ts b/src/panel/setupPanel.ts index 901823d..2aba18c 100644 --- a/src/panel/setupPanel.ts +++ b/src/panel/setupPanel.ts @@ -79,9 +79,7 @@ export async function setupPanel(operationState: OperationState): Promise= 20): maxK = floor(N / 3). + * Ensures each cluster has at least ~3 notes on average. + * This is more generous than sqrt(N) and prevents under-clustering + * (e.g. N=56 → [2,15] instead of [2,7]). + * - maxK is always clamped to MAX_K_CAP (15). + * + * @param n Number of data points + * @returns Tuple [minK, maxK] + */ +export function computeKRange(n: number): [number, number] { + if (n < 2) return [1, 1]; // degenerate: can't cluster at all + + const minK = MIN_K; + let maxK: number; + + if (n < 20) { + // For small datasets, allow up to N/2 clusters so the sweep + // can actually explore meaningful K values (e.g. N=8 → [2,4]) + maxK = Math.max(MIN_K, Math.floor(n / 2)); + } else { + // For larger datasets, allow 1 cluster per 3 notes on average. + // This is more generous than sqrt(N) and avoids under-clustering + // (e.g. N=56 → maxK=18 capped to 15, vs sqrt giving only 7). + maxK = Math.floor(n / 3); + } + + // Clamp to [minK, MAX_K_CAP] + maxK = Math.min(Math.max(maxK, minK), MAX_K_CAP); + + return [minK, maxK]; +} + +/** + * Finds the optimal number of clusters (K) by sweeping K values and + * using silhouette-tolerance selection. + * + * Algorithm: + * 1. Run the specified clustering algorithm for each K in [minK, maxK]. + * 2. Compute silhouette scores for all valid K values. + * 3. Find the peak silhouette score across the sweep. + * 4. Among all K values whose silhouette is within SILHOUETTE_TOLERANCE + * of the peak, select the **highest K**. + * + * This tolerance-based approach prevents the well-known silhouette bias + * toward coarse clusters. When scores are nearly identical (e.g. K=4 at + * 0.551 vs K=7 at 0.533), the finer granularity is preferred because it + * produces more useful note categories. + * + * @param vectors Input data points (N x D), already UMAP-reduced if applicable + * @param algorithm Which algorithm to use: 'kmeans' or 'kmedoids' + * @param distFn Distance function (cosine or euclidean) + * @param seed Seed for reproducible initialization + * @returns The optimal K, its assignments, and its silhouette score + */ +export function findOptimalK( + vectors: number[][], + algorithm: 'kmeans' | 'kmedoids', + distFn: DistanceFn, + seed: number, +): AutoKResult { + const n = vectors.length; + const [minK, maxK] = computeKRange(n); + + log(`Auto-K: sweeping K=${minK}..${maxK} for ${algorithm} (N=${n})`); + + const clusterFn = algorithm === 'kmeans' ? kmeans : kmedoids; + + // Collect all valid (k, score, assignments) candidates + const candidates: { k: number; score: number; assignments: number[] }[] = []; + + for (let k = minK; k <= maxK; k++) { + const assignments = clusterFn(vectors, k, distFn, seed + k); + + // Count unique non-negative clusters actually formed + const uniqueClusters = new Set(assignments.filter((a) => a >= 0)); + + // Silhouette requires at least 2 distinct clusters + if (uniqueClusters.size < 2) { + log(` K=${k}: only ${uniqueClusters.size} cluster(s) formed, skipping`); + continue; + } + + const score = silhouetteScore(vectors, assignments, distFn); + log(` K=${k}: silhouette=${score.toFixed(4)}`); + candidates.push({ k, score, assignments }); + } + + let bestK: number; + let bestScore: number; + let bestAssignments: number[]; + + if (candidates.length === 0) { + // Fallback: no K produced ≥2 valid clusters, put everything in one cluster + log('Auto-K: no valid clustering found, falling back to single cluster (K=1)'); + bestK = 1; + bestAssignments = new Array(n).fill(0); + bestScore = 0; + } else { + // Find peak silhouette across all candidates + const peakScore = Math.max(...candidates.map((c) => c.score)); + const threshold = peakScore - SILHOUETTE_TOLERANCE; + + // Among candidates within tolerance of peak, pick the highest K. + // This prevents silhouette's natural bias toward coarser clusters. + const viable = candidates.filter((c) => c.score >= threshold); + const best = viable.reduce((a, b) => (a.k >= b.k ? a : b)); + + bestK = best.k; + bestScore = best.score; + bestAssignments = best.assignments; + } + + log(`Auto-K: best K=${bestK} (silhouette=${bestScore.toFixed(4)})`); + + return { + bestK, + assignments: bestAssignments, + silhouetteScore: bestScore, + }; +} diff --git a/src/pipeline/clustering/benchmark.ts b/src/pipeline/clustering/benchmark.ts index 5d87b0e..fd99cf0 100644 --- a/src/pipeline/clustering/benchmark.ts +++ b/src/pipeline/clustering/benchmark.ts @@ -1,8 +1,9 @@ import { CategorizationConfig, BenchmarkResult, ClusteringStrategy } from '../../types/cluster'; -import { DistanceFn, getDistanceFn, silhouetteScore } from './metrics'; +import { DistanceFn, getDistanceFn, silhouetteScore, euclideanDistance } from './metrics'; import { kmeans } from './kmeans'; import { kmedoids } from './kmedoids'; import { hdbscan } from './hdbscan'; +import { findOptimalK } from './autoK'; import { UmapProjector } from '../UmapProjector'; import { log } from '../../utils/logger'; @@ -12,7 +13,15 @@ const DEFAULT_MIN_CLUSTER_SIZE = 3; /** * Runs a single clustering strategy and returns the cluster assignments. */ -function runStrategy(vectors: number[][], strategy: ClusteringStrategy, distFn: DistanceFn, seed: number): number[] { +export function runStrategy( + vectors: number[][], + strategy: ClusteringStrategy, + distFn: DistanceFn, + seed: number, +): number[] { + if (strategy.K === 'auto') { + throw new Error(`runStrategy called with K='auto' for ${strategy.algorithm}. Use findOptimalK() instead.`); + } switch (strategy.algorithm) { case 'kmeans': return kmeans(vectors, strategy.K ?? DEFAULT_K, distFn, seed); @@ -119,6 +128,18 @@ export function benchmark( clusteringVectors = projector.project(vectors); } + // UMAP output coordinates live in Euclidean space, so clustering and + // silhouette evaluation must use Euclidean distance regardless of + // the metric used by UMAP internally to build its neighborhood graph. + const clusterDistFn: DistanceFn = (distanceMatrix || config.intermediateDim !== null) + ? euclideanDistance + : distFn; + + const metricName = (distanceMatrix || config.intermediateDim !== null) + ? `euclidean (UMAP ${clusteringVectors[0]?.length ?? 0}D space)` + : config.metric; + log(`Clustering metric: using ${metricName} distance`); + const results: BenchmarkResult[] = []; for (const strategy of config.strategies) { @@ -126,20 +147,31 @@ export function benchmark( const startTime = performance.now(); try { - const assignments = runStrategy(clusteringVectors, strategy, distFn, config.seed); + let assignments: number[]; + let score: number; + + if (strategy.K === 'auto' && (strategy.algorithm === 'kmeans' || strategy.algorithm === 'kmedoids')) { + // Auto-K: sweep K range and pick the best + const autoResult = findOptimalK(clusteringVectors, strategy.algorithm, clusterDistFn, config.seed); + assignments = autoResult.assignments; + score = autoResult.silhouetteScore; + } else { + assignments = runStrategy(clusteringVectors, strategy, clusterDistFn, config.seed); + score = 0; + } + const timeMs = performance.now() - startTime; const outlierCount = assignments.filter((a) => a < 0).length; const clusterSizes = computeClusterSizes(assignments); const clusterCount = clusterSizes.filter((s) => s > 0).length; - // For silhouette, exclude noise points (-1) since they're intentionally unassigned - let score = 0; - if (clusterCount >= 2) { + // Compute silhouette if not already computed by auto-K + if (strategy.K !== 'auto' && clusterCount >= 2) { const clusteredIndices = assignments.map((a, i) => (a >= 0 ? i : -1)).filter((i) => i >= 0); const clusteredVectors = clusteredIndices.map((i) => clusteringVectors[i]); const clusteredAssignments = clusteredIndices.map((i) => assignments[i]); - score = silhouetteScore(clusteredVectors, clusteredAssignments, distFn); + score = silhouetteScore(clusteredVectors, clusteredAssignments, clusterDistFn); } results.push({ diff --git a/src/pipeline/pipelineConfig.ts b/src/pipeline/pipelineConfig.ts index 16bf758..87e57be 100644 --- a/src/pipeline/pipelineConfig.ts +++ b/src/pipeline/pipelineConfig.ts @@ -1,4 +1,4 @@ -import { CategorizationConfig } from '../types/cluster'; +import { CategorizationConfig, MetricType } from '../types/cluster'; /** Default dimensionality of local ONNX embedding vectors (all-MiniLM-L6-v2 / multilingual-e5-small). */ export const EMBEDDING_DIM = 384; @@ -29,28 +29,40 @@ export function adaptiveNeighbors(noteCount: number): number { return Math.max(5, Math.min(50, raw)); } -export function createAdaptiveConfig(inputDim: number, noteCount: number): CategorizationConfig { +export function createAdaptiveConfig( + inputDim: number, + noteCount: number, + metric: MetricType = 'cosine', + seed = 42, +): CategorizationConfig { return { - seed: 42, - metric: 'cosine', + seed, + metric, intermediateDim: adaptiveIntermediateDim(inputDim), intermediateNeighbors: adaptiveNeighbors(noteCount), strategies: [ - { name: 'kmeans-6', algorithm: 'kmeans', K: 6 }, - { name: 'kmedoids-6', algorithm: 'kmedoids', K: 6 }, + { name: 'kmeans-auto', algorithm: 'kmeans', K: 'auto' }, + { name: 'kmedoids-auto', algorithm: 'kmedoids', K: 'auto' }, { name: 'hdbscan', algorithm: 'hdbscan', minClusterSize: 3, minSamples: 2 }, ], }; } -export const DEFAULT_CONFIG: CategorizationConfig = { - seed: 42, - metric: 'cosine', - intermediateDim: 8, - intermediateNeighbors: 5, - strategies: [ - { name: 'kmeans-6', algorithm: 'kmeans', K: 6 }, - { name: 'kmedoids-6', algorithm: 'kmedoids', K: 6 }, - { name: 'hdbscan', algorithm: 'hdbscan', minClusterSize: 3, minSamples: 2 }, - ], -}; +export function createPipelineConfig( + metric: MetricType = 'cosine', + seed = 42, +): CategorizationConfig { + return { + seed, + metric, + intermediateDim: 8, + intermediateNeighbors: 5, + strategies: [ + { name: 'kmeans-auto', algorithm: 'kmeans', K: 'auto' }, + { name: 'kmedoids-auto', algorithm: 'kmedoids', K: 'auto' }, + { name: 'hdbscan', algorithm: 'hdbscan', minClusterSize: 3, minSamples: 2 }, + ], + }; +} + +export const DEFAULT_CONFIG: CategorizationConfig = createPipelineConfig(); diff --git a/src/pipeline/runPipeline.ts b/src/pipeline/runPipeline.ts index 45339bf..43d298e 100644 --- a/src/pipeline/runPipeline.ts +++ b/src/pipeline/runPipeline.ts @@ -2,10 +2,11 @@ import { fetchAllNotes } from './noteReader'; import { benchmark } from './clustering/benchmark'; import { weightedAverageVectorsWithNorm } from './vectorAggregator'; import { PanelNote } from '../types/panel'; +import { MetricType } from '../types/cluster'; import { log, logErr } from '../utils/logger'; import { VectorCache } from './vectorCache'; import { isNativeAiReady, fetchNativeEmbeddings } from './nativeEmbeddingPipeline'; -import { DEFAULT_CONFIG, isValidEmbeddingVector, createAdaptiveConfig } from './pipelineConfig'; +import { createPipelineConfig, isValidEmbeddingVector, createAdaptiveConfig } from './pipelineConfig'; import { enrichResultsWithTags } from './clustering/postProcess'; import { upgradeClusterNamesWithAi } from './clustering/aiNamingService'; import { EmbeddingWorkerOrchestrator } from './EmbeddingWorkerOrchestrator'; @@ -44,6 +45,11 @@ export const runPipeline = async (installDir: string, callbacks: PipelineCallbac return; } + // Pipeline configuration defaults (Cosine metric for text embeddings, seed 42 for reproducibility) + const userMetric: MetricType = 'cosine'; + const userSeed = 42; + log(`Pipeline settings: metric="${userMetric}", seed=${userSeed}`); + if (await isNativeAiReady()) { log('Native AI Search active: using native embeddings pipeline'); callbacks.onStatus('Fetching native embeddings...'); @@ -97,7 +103,7 @@ export const runPipeline = async (installDir: string, callbacks: PipelineCallbac log('Too few indexed notes found in native DB. Falling back to local ONNX Web Worker.'); } else { callbacks.onStatus('Clustering...'); - const adaptiveConfig = createAdaptiveConfig(nativeResult.dimension, validNotes.length); + const adaptiveConfig = createAdaptiveConfig(nativeResult.dimension, validNotes.length, userMetric, userSeed); const results = benchmark(vectors, adaptiveConfig); // Post-process to extract tags/keywords for each cluster (keep parity with local pipeline) @@ -172,7 +178,8 @@ export const runPipeline = async (installDir: string, callbacks: PipelineCallbac } const vectors = noteVectors.map((nv) => nv.vector); - const results = benchmark(vectors, DEFAULT_CONFIG); + const pipelineConfig = createPipelineConfig(userMetric, userSeed); + const results = benchmark(vectors, pipelineConfig); // Post-process to extract tags/keywords for each cluster const notesMap = new Map(notes.map((n) => [n.id, n])); diff --git a/src/pipeline/vectorAggregator.ts b/src/pipeline/vectorAggregator.ts index 6f45b47..d803b14 100644 --- a/src/pipeline/vectorAggregator.ts +++ b/src/pipeline/vectorAggregator.ts @@ -128,3 +128,36 @@ export const blendVectors = (body: number[], title: number[], alpha: number): nu } return normalise(blended); }; + +/** + * Computes a weighted average of chunk vectors where chunk 0 (which contains the title) + * is given a higher weight (default: 3.0) to prevent dilution. + */ +export const averageChunksWeighted = ( + chunks: { chunkIndex: number; vector: number[] }[], + titleWeight = 3.0, +): number[] => { + if (chunks.length === 0) throw new Error('Cannot average zero chunks'); + const dim = chunks[0].vector.length; + for (const chunk of chunks) { + if (chunk.vector.length !== dim) throw new Error('Cannot average vectors of different dimensions'); + } + if (chunks.length === 1) { + return normalise(chunks[0].vector); + } + + const sum = new Array(dim).fill(0); + let totalWeight = 0; + for (const chunk of chunks) { + const weight = chunk.chunkIndex === 0 ? titleWeight : 1.0; + totalWeight += weight; + for (let i = 0; i < dim; i++) { + sum[i] += chunk.vector[i] * weight; + } + } + for (let i = 0; i < dim; i++) { + sum[i] /= totalWeight; + } + + return normalise(sum); +}; diff --git a/src/settings/registerSettings.ts b/src/settings/registerSettings.ts index 319ac7a..a3a6551 100644 --- a/src/settings/registerSettings.ts +++ b/src/settings/registerSettings.ts @@ -55,9 +55,7 @@ export async function runNativeCleanup(source: string, operationState: Operation await joplin.views.dialogs.showMessageBox(lastMessage); } } catch (err) { - await joplin.views.dialogs.showMessageBox( - `Cleanup failed: ${err instanceof Error ? err.message : String(err)}`, - ); + await joplin.views.dialogs.showMessageBox(`Cleanup failed: ${err instanceof Error ? err.message : String(err)}`); } finally { operationState.inProgress = false; } @@ -71,35 +69,13 @@ export async function registerPluginSettings(operationState: OperationState): Pr }); await joplin.settings.registerSettings({ - 'categorization.metric': { - value: 'cosine', - type: SettingType.String, - isEnum: true, - options: { - cosine: 'Cosine Similarity (Recommended)', - euclidean: 'Euclidean Distance', - }, - section: 'aiCategorization', - public: true, - label: 'Distance Metric', - description: 'Distance metric used for note embedding comparisons and clustering.', - }, 'categorization.parentNotebook': { value: '', type: SettingType.String, section: 'aiCategorization', public: true, label: 'Default Target Notebook', - description: - 'Default parent notebook where newly categorized sub-notebooks will be created (leave empty for root).', - }, - 'categorization.seed': { - value: 42, - type: SettingType.Int, - section: 'aiCategorization', - public: true, - label: 'Random Seed', - description: 'Random seed for reproducible UMAP projections and K-Means clustering.', + description: 'Default parent notebook where newly categorized sub-notebooks will be created (leave empty for root).', }, 'categorization.changeLog': { value: '', @@ -123,8 +99,7 @@ export async function registerPluginSettings(operationState: OperationState): Pr section: 'aiCategorization', public: true, label: 'Undo Last Categorization', - description: - 'Check this box and click Apply/OK to revert note movements and tags from the previous run.', + description: 'Check this box and click Apply/OK to revert note movements and tags from the previous run.', }, 'categorization.cleanUpAction': { value: false, @@ -132,8 +107,7 @@ export async function registerPluginSettings(operationState: OperationState): Pr section: 'aiCategorization', public: true, label: 'Clean Up Empty Notebooks', - description: - 'Check this box and click Apply/OK to check for and remove empty notebooks leftover after note moves.', + description: 'Check this box and click Apply/OK to check for and remove empty notebooks leftover after note moves.', }, }); diff --git a/src/types/cluster.ts b/src/types/cluster.ts index a2b9271..db1a982 100644 --- a/src/types/cluster.ts +++ b/src/types/cluster.ts @@ -4,19 +4,21 @@ export interface ClusteringStrategy { /** Human-readable label for this run, e.g. 'kmeans-5' */ name: string; algorithm: ClusteringAlgorithm; - /** Number of clusters (kmeans / kmedoids) */ - K?: number; + /** Number of clusters (kmeans / kmedoids). Use 'auto' for automatic selection via silhouette sweep. */ + K?: number | 'auto'; /** Minimum points to form a cluster (hdbscan, default: 3) */ minClusterSize?: number; /** How many neighbors define a "core" point (hdbscan, default: minClusterSize). Lower = fewer outliers */ minSamples?: number; } +export type MetricType = 'cosine' | 'euclidean'; + export interface CategorizationConfig { /** Seed for UMAP and clustering reproducibility */ seed: number; /** Distance metric for clustering and UMAP */ - metric: 'cosine' | 'euclidean'; + metric: MetricType; /** * If set, UMAP-reduce to this dimensionality before clustering. * null = cluster directly on the raw embedding vectors (e.g. 384D). diff --git a/src/types/panel.ts b/src/types/panel.ts index 81e3f42..8f979cc 100644 --- a/src/types/panel.ts +++ b/src/types/panel.ts @@ -51,7 +51,12 @@ export type WebviewMessage = | { type: 'run' } | { type: 'poll' } | { type: 'getInitialState' } - | { type: 'syncState'; strategies: BenchmarkResult[]; notes: PanelNote[]; selectedStrategyIndex: number } + | { + type: 'syncState'; + strategies: BenchmarkResult[]; + notes: PanelNote[]; + selectedStrategyIndex: number; + } | { type: 'openNote'; noteId: string } | { type: 'getSettings' } | { type: 'updateSetting'; key: string; value: string } diff --git a/src/webview/context/AppStateContext.tsx b/src/webview/context/AppStateContext.tsx index ba41c17..f4ff612 100644 --- a/src/webview/context/AppStateContext.tsx +++ b/src/webview/context/AppStateContext.tsx @@ -283,8 +283,7 @@ export const AppStateProvider: React.FC<{ children: React.ReactNode }> = ({ chil React.useEffect(() => { fetchSettings(); if (typeof webviewApi !== 'undefined') { - webviewApi - .postMessage({ type: 'getInitialState' }) + webviewApi.postMessage({ type: 'getInitialState' }) .then((initialState) => { if (initialState) { handlePollResponse(initialState); @@ -301,16 +300,14 @@ export const AppStateProvider: React.FC<{ children: React.ReactNode }> = ({ chil React.useEffect(() => { if (typeof webviewApi !== 'undefined' && strategies && strategies.length > 0) { - webviewApi - .postMessage({ - type: 'syncState', - strategies, - notes, - selectedStrategyIndex, - }) - .catch((err) => { - console.error('syncState error:', err); - }); + webviewApi.postMessage({ + type: 'syncState', + strategies, + notes, + selectedStrategyIndex, + }).catch((err) => { + console.error('syncState error:', err); + }); } }, [strategies, notes, selectedStrategyIndex]); diff --git a/src/webview/context/useApplyState.ts b/src/webview/context/useApplyState.ts index f8b2a47..112bb5e 100644 --- a/src/webview/context/useApplyState.ts +++ b/src/webview/context/useApplyState.ts @@ -26,7 +26,11 @@ export function useApplyState(startPolling: () => void) { }, []); const applyChanges = React.useCallback( - async (options: ApplyOptions, notes: PanelNote[], currentStrategy: BenchmarkResult | undefined) => { + async ( + options: ApplyOptions, + notes: PanelNote[], + currentStrategy: BenchmarkResult | undefined, + ) => { if (!currentStrategy) { setApplyError('No active strategy selected.'); return; diff --git a/src/webview/context/useSettingsState.ts b/src/webview/context/useSettingsState.ts index ad220a0..ac2a04e 100644 --- a/src/webview/context/useSettingsState.ts +++ b/src/webview/context/useSettingsState.ts @@ -1,17 +1,13 @@ import * as React from 'react'; interface SettingsResponse { - 'categorization.metric': string; 'categorization.parentNotebook': string; - 'categorization.seed': number; 'categorization.changeLog': string; } export function useSettingsState() { const [settings, setSettings] = React.useState({ - metric: 'cosine', parentNotebook: '', - seed: 42, changeLog: '', }); @@ -22,9 +18,7 @@ export function useSettingsState() { if (res) { const data = res as unknown as SettingsResponse; setSettings({ - metric: data['categorization.metric'] || 'cosine', parentNotebook: data['categorization.parentNotebook'] || '', - seed: data['categorization.seed'] ?? 42, changeLog: data['categorization.changeLog'] || '', }); } diff --git a/src/webview/pages/SettingsPage.tsx b/src/webview/pages/SettingsPage.tsx index 54cfa47..5f771c1 100644 --- a/src/webview/pages/SettingsPage.tsx +++ b/src/webview/pages/SettingsPage.tsx @@ -8,17 +8,14 @@ export const SettingsPage: React.FC = () => {
Plugin Settings
- Settings are managed in Joplin's native Options window. Go to{' '} - Tools → Options → AI Categorization to configure the plugin. + Settings are managed in Joplin's native Options window. + Go to Tools → Options → AI Categorization to configure the plugin.
-
+
Active Configuration:
- • Distance Metric: {settings.metric || 'Cosine Similarity'} + • Distance Metric: Cosine Similarity
• Target Notebook: {settings.parentNotebook || '(Root Notebooks)'} diff --git a/src/webview/panel.tsx b/src/webview/panel.tsx index f08b14a..18de726 100644 --- a/src/webview/panel.tsx +++ b/src/webview/panel.tsx @@ -11,7 +11,9 @@ const AppContent: React.FC = () => {
{error &&
Error: {error}
} -
{activeView === 'idle' ? : }
+
+ {activeView === 'idle' ? : } +
); }; diff --git a/test/pipeline/clustering/autoK.test.ts b/test/pipeline/clustering/autoK.test.ts new file mode 100644 index 0000000..7b86001 --- /dev/null +++ b/test/pipeline/clustering/autoK.test.ts @@ -0,0 +1,228 @@ +import { computeKRange, findOptimalK } from '../../../src/pipeline/clustering/autoK'; +import { benchmark, runStrategy } from '../../../src/pipeline/clustering/benchmark'; +import { euclideanDistance } from '../../../src/pipeline/clustering/metrics'; +import { CategorizationConfig } from '../../../src/types/cluster'; + +// Synthetic 3-cluster data: 30 points in 2D +const THREE_CLUSTERS = [ + // Cluster near [0, 0] + [0.1, 0.2], + [0.2, 0.1], + [0.0, 0.0], + [0.1, 0.1], + [0.15, 0.15], + [0.2, 0.2], + [0.05, 0.05], + [0.12, 0.08], + [0.08, 0.12], + [0.18, 0.11], + // Cluster near [100, 0] + [100.1, 0.2], + [99.9, 0.1], + [100.0, 0.0], + [100.1, 0.1], + [100.15, 0.15], + [99.95, 0.2], + [100.05, 0.05], + [100.12, 0.08], + [100.08, 0.12], + [100.18, 0.11], + // Cluster near [0, 100] + [0.1, 100.2], + [0.2, 99.9], + [0.0, 100.0], + [0.1, 100.1], + [0.15, 100.15], + [0.2, 100.2], + [0.05, 100.05], + [0.12, 100.08], + [0.08, 100.12], + [0.18, 100.11], +]; + +// Synthetic 2-cluster data: 30 points in 2D +const TWO_CLUSTERS = [ + // Cluster near [0, 0] + [0.1, 0.2], + [0.2, 0.1], + [0.0, 0.0], + [0.1, 0.1], + [0.15, 0.15], + [0.2, 0.2], + [0.05, 0.05], + [0.12, 0.08], + [0.08, 0.12], + [0.18, 0.11], + [0.02, 0.03], + [0.04, 0.05], + [0.11, 0.09], + [0.14, 0.12], + [0.07, 0.06], + // Cluster near [100, 0] + [100.1, 0.2], + [99.9, 0.1], + [100.0, 0.0], + [100.1, 0.1], + [100.15, 0.15], + [99.95, 0.2], + [100.05, 0.05], + [100.12, 0.08], + [100.08, 0.12], + [100.18, 0.11], + [100.02, 0.03], + [100.04, 0.05], + [100.11, 0.09], + [100.14, 0.12], + [100.07, 0.06], +]; + +describe('autoK computeKRange', () => { + it('handles degenerate cases correctly', () => { + expect(computeKRange(0)).toEqual([1, 1]); + expect(computeKRange(1)).toEqual([1, 1]); + }); + + it('computes correct range for very small datasets (N < 6)', () => { + expect(computeKRange(2)).toEqual([2, 2]); + expect(computeKRange(3)).toEqual([2, 2]); + expect(computeKRange(4)).toEqual([2, 2]); + expect(computeKRange(5)).toEqual([2, 2]); + }); + + it('computes correct range for small-to-medium datasets (N < 20, uses N/2)', () => { + expect(computeKRange(6)).toEqual([2, 3]); // floor(6/2) = 3 + expect(computeKRange(8)).toEqual([2, 4]); // floor(8/2) = 4 + expect(computeKRange(9)).toEqual([2, 4]); // floor(9/2) = 4 + expect(computeKRange(10)).toEqual([2, 5]); // floor(10/2) = 5 + expect(computeKRange(12)).toEqual([2, 6]); // floor(12/2) = 6 + expect(computeKRange(19)).toEqual([2, 9]); // floor(19/2) = 9 + }); + + it('computes correct range for larger datasets (N >= 20, uses N/3)', () => { + expect(computeKRange(20)).toEqual([2, 6]); // floor(20/3) = 6 + expect(computeKRange(30)).toEqual([2, 10]); // floor(30/3) = 10 + expect(computeKRange(45)).toEqual([2, 15]); // floor(45/3) = 15, hits cap + expect(computeKRange(56)).toEqual([2, 15]); // floor(56/3) = 18, capped at 15 + expect(computeKRange(100)).toEqual([2, 15]); // floor(100/3) = 33, capped at 15 + expect(computeKRange(500)).toEqual([2, 15]); // capped at MAX_K_CAP=15 + }); +}); + +describe('autoK findOptimalK', () => { + it('identifies 3 well-separated clusters', () => { + const result = findOptimalK(THREE_CLUSTERS, 'kmeans', euclideanDistance, 42); + expect(result.bestK).toBe(3); + expect(result.silhouetteScore).toBeGreaterThan(0.9); + expect(result.assignments).toHaveLength(THREE_CLUSTERS.length); + }); + + it('identifies 2 well-separated clusters', () => { + const result = findOptimalK(TWO_CLUSTERS, 'kmeans', euclideanDistance, 42); + expect(result.bestK).toBe(2); + expect(result.silhouetteScore).toBeGreaterThan(0.9); + expect(result.assignments).toHaveLength(TWO_CLUSTERS.length); + }); + + it('returns K=1 when data has no clear structure / identical points (fallback)', () => { + // All points are identical -> silhouette scores are invalid/cannot form 2 clusters -> falls back to K=1 + const identicalPoints = Array.from({ length: 10 }, () => [1.0, 1.0]); + const result = findOptimalK(identicalPoints, 'kmeans', euclideanDistance, 42); + expect(result.bestK).toBe(1); + expect(result.assignments).toEqual(new Array(10).fill(0)); + }); + + it('is deterministic when given the same seed', () => { + const res1 = findOptimalK(THREE_CLUSTERS, 'kmeans', euclideanDistance, 42); + const res2 = findOptimalK(THREE_CLUSTERS, 'kmeans', euclideanDistance, 42); + expect(res1).toEqual(res2); + }); + + it('works with both kmeans and kmedoids', () => { + const resKmeans = findOptimalK(THREE_CLUSTERS, 'kmeans', euclideanDistance, 42); + const resKmedoids = findOptimalK(THREE_CLUSTERS, 'kmedoids', euclideanDistance, 42); + expect(resKmeans.bestK).toBe(3); + expect(resKmedoids.bestK).toBe(3); + }); + + it('falls back to K=1 when no valid clustering is possible', () => { + // Single point: can't form 2 clusters + const result = findOptimalK([[1.0, 2.0]], 'kmeans', euclideanDistance, 42); + expect(result.bestK).toBe(1); + expect(result.assignments).toEqual([0]); + expect(result.silhouetteScore).toBe(0); + }); + + it('prefers higher K when silhouette scores are within tolerance', () => { + // 4 clusters of 8 points each, reasonably separated in 2D. + // K=2 and K=3 may score slightly higher in raw silhouette, but K=4 + // should be within the 0.025 tolerance band and thus selected. + const FOUR_CLUSTERS = [ + // Cluster near [0, 0] + ...[0.1, 0.2, 0.0, 0.15, 0.05, 0.12, 0.08, 0.18].map((x, i) => [x, [0.2, 0.1, 0.0, 0.15, 0.05, 0.08, 0.12, 0.11][i]]), + // Cluster near [10, 0] + ...[10.1, 10.2, 10.0, 10.15, 10.05, 10.12, 10.08, 10.18].map((x, i) => [x, [0.2, 0.1, 0.0, 0.15, 0.05, 0.08, 0.12, 0.11][i]]), + // Cluster near [0, 10] + ...[0.1, 0.2, 0.0, 0.15, 0.05, 0.12, 0.08, 0.18].map((x, i) => [x, 10 + [0.2, 0.1, 0.0, 0.15, 0.05, 0.08, 0.12, 0.11][i]]), + // Cluster near [10, 10] + ...[10.1, 10.2, 10.0, 10.15, 10.05, 10.12, 10.08, 10.18].map((x, i) => [x, 10 + [0.2, 0.1, 0.0, 0.15, 0.05, 0.08, 0.12, 0.11][i]]), + ]; + + const result = findOptimalK(FOUR_CLUSTERS, 'kmeans', euclideanDistance, 42); + // With tolerance-based selection, K=4 should be picked even if K=2 or K=3 + // has a marginally higher raw silhouette + expect(result.bestK).toBeGreaterThanOrEqual(3); + expect(result.silhouetteScore).toBeGreaterThan(0.5); + expect(result.assignments).toHaveLength(FOUR_CLUSTERS.length); + }); +}); + +describe('benchmark integration with autoK', () => { + it('runs auto-K strategy and produces valid BenchmarkResult', () => { + const config: CategorizationConfig = { + seed: 42, + metric: 'euclidean', + intermediateDim: null, + intermediateNeighbors: 5, + strategies: [{ name: 'kmeans-auto', algorithm: 'kmeans', K: 'auto' }], + }; + + const results = benchmark(THREE_CLUSTERS, config); + expect(results).toHaveLength(1); + expect(results[0].strategyName).toBe('kmeans-auto'); + expect(results[0].algorithm).toBe('kmeans'); + expect(results[0].clusterCount).toBe(3); + expect(results[0].silhouetteScore).toBeGreaterThan(0.9); + expect(results[0].assignments).toHaveLength(THREE_CLUSTERS.length); + expect(results[0].outlierCount).toBe(0); + }); + + it('handles mixed auto and fixed strategies', () => { + const config: CategorizationConfig = { + seed: 42, + metric: 'euclidean', + intermediateDim: null, + intermediateNeighbors: 5, + strategies: [ + { name: 'kmeans-auto', algorithm: 'kmeans', K: 'auto' }, + { name: 'kmeans-5', algorithm: 'kmeans', K: 5 }, + { name: 'hdbscan', algorithm: 'hdbscan', minClusterSize: 3, minSamples: 2 }, + ], + }; + + const results = benchmark(THREE_CLUSTERS, config); + expect(results).toHaveLength(3); + + // The results should be sorted by silhouetteScore descending + expect(results[0].silhouetteScore).toBeGreaterThanOrEqual(results[1].silhouetteScore); + expect(results[1].silhouetteScore).toBeGreaterThanOrEqual(results[2].silhouetteScore); + + const autoRes = results.find((r) => r.strategyName === 'kmeans-auto')!; + expect(autoRes.clusterCount).toBe(3); + }); + + it('runStrategy throws an error when called directly with K="auto"', () => { + expect(() => { + runStrategy(THREE_CLUSTERS, { name: 'kmeans-auto', algorithm: 'kmeans', K: 'auto' }, euclideanDistance, 42); + }).toThrow('Use findOptimalK() instead'); + }); +}); diff --git a/test/pipeline/vectorAggregator.test.ts b/test/pipeline/vectorAggregator.test.ts index b9dc06f..95b49fe 100644 --- a/test/pipeline/vectorAggregator.test.ts +++ b/test/pipeline/vectorAggregator.test.ts @@ -5,6 +5,7 @@ import { cosineSimilarity, computeTitleWeight, blendVectors, + averageChunksWeighted, } from '../../src/pipeline/vectorAggregator'; describe('averageVectors', () => { @@ -189,3 +190,55 @@ describe('blendVectors', () => { expect(() => blendVectors([1, 2], [1, 2, 3], 0.5)).toThrow('different dimensions'); }); }); + +describe('averageChunksWeighted', () => { + it('normalizes a single chunk', () => { + const result = averageChunksWeighted([{ chunkIndex: 0, vector: [3, 4] }]); + expect(result[0]).toBeCloseTo(0.6, 10); + expect(result[1]).toBeCloseTo(0.8, 10); + }); + + it('computes weighted average where chunk 0 gets higher weight', () => { + // Chunk 0 has vector [1, 0], chunk 1 has vector [0, 1] + // With default titleWeight = 3.0: + // sum = 3.0 * [1, 0] + 1.0 * [0, 1] = [3.0, 1.0] + // norm = sqrt(9 + 1) = sqrt(10) ≈ 3.16227766 + // normalized = [3/√10, 1/√10] + const result = averageChunksWeighted([ + { chunkIndex: 0, vector: [1, 0] }, + { chunkIndex: 1, vector: [0, 1] }, + ]); + const norm = Math.sqrt(10); + expect(result[0]).toBeCloseTo(3 / norm, 10); + expect(result[1]).toBeCloseTo(1 / norm, 10); + }); + + it('uses custom titleWeight', () => { + // Custom titleWeight = 5.0 + // sum = 5.0 * [1, 0] + 1.0 * [0, 1] = [5.0, 1.0] + // norm = sqrt(25 + 1) = sqrt(26) + const result = averageChunksWeighted( + [ + { chunkIndex: 0, vector: [1, 0] }, + { chunkIndex: 1, vector: [0, 1] }, + ], + 5.0, + ); + const norm = Math.sqrt(26); + expect(result[0]).toBeCloseTo(5 / norm, 10); + expect(result[1]).toBeCloseTo(1 / norm, 10); + }); + + it('throws on empty input', () => { + expect(() => averageChunksWeighted([])).toThrow('Cannot average zero chunks'); + }); + + it('throws on dimension mismatch', () => { + expect(() => + averageChunksWeighted([ + { chunkIndex: 0, vector: [1, 2] }, + { chunkIndex: 1, vector: [1, 2, 3] }, + ]), + ).toThrow('different dimensions'); + }); +}); From f9879fcee86ca04f32dfd5fd95a4451126408473 Mon Sep 17 00:00:00 2001 From: Harsh16gupta Date: Wed, 5 Aug 2026 10:28:28 +0530 Subject: [PATCH 3/3] fix(ci): remove unused metric property from AppStateContextType --- src/pipeline/clustering/benchmark.ts | 11 ++++---- src/pipeline/pipelineConfig.ts | 5 +--- src/pipeline/runPipeline.ts | 7 ++++- src/settings/registerSettings.ts | 13 +++++++--- src/types/panel.ts | 7 +---- src/webview/context/AppStateContext.tsx | 22 ++++++++-------- src/webview/context/useApplyState.ts | 6 +---- src/webview/pages/SettingsPage.tsx | 9 ++++--- src/webview/panel.tsx | 4 +-- test/pipeline/clustering/autoK.test.ts | 34 +++++++++++++++++-------- 10 files changed, 65 insertions(+), 53 deletions(-) diff --git a/src/pipeline/clustering/benchmark.ts b/src/pipeline/clustering/benchmark.ts index fd99cf0..8ec7f43 100644 --- a/src/pipeline/clustering/benchmark.ts +++ b/src/pipeline/clustering/benchmark.ts @@ -131,13 +131,12 @@ export function benchmark( // UMAP output coordinates live in Euclidean space, so clustering and // silhouette evaluation must use Euclidean distance regardless of // the metric used by UMAP internally to build its neighborhood graph. - const clusterDistFn: DistanceFn = (distanceMatrix || config.intermediateDim !== null) - ? euclideanDistance - : distFn; + const clusterDistFn: DistanceFn = distanceMatrix || config.intermediateDim !== null ? euclideanDistance : distFn; - const metricName = (distanceMatrix || config.intermediateDim !== null) - ? `euclidean (UMAP ${clusteringVectors[0]?.length ?? 0}D space)` - : config.metric; + const metricName = + distanceMatrix || config.intermediateDim !== null + ? `euclidean (UMAP ${clusteringVectors[0]?.length ?? 0}D space)` + : config.metric; log(`Clustering metric: using ${metricName} distance`); const results: BenchmarkResult[] = []; diff --git a/src/pipeline/pipelineConfig.ts b/src/pipeline/pipelineConfig.ts index 87e57be..b9f5a62 100644 --- a/src/pipeline/pipelineConfig.ts +++ b/src/pipeline/pipelineConfig.ts @@ -48,10 +48,7 @@ export function createAdaptiveConfig( }; } -export function createPipelineConfig( - metric: MetricType = 'cosine', - seed = 42, -): CategorizationConfig { +export function createPipelineConfig(metric: MetricType = 'cosine', seed = 42): CategorizationConfig { return { seed, metric, diff --git a/src/pipeline/runPipeline.ts b/src/pipeline/runPipeline.ts index 43d298e..0196963 100644 --- a/src/pipeline/runPipeline.ts +++ b/src/pipeline/runPipeline.ts @@ -103,7 +103,12 @@ export const runPipeline = async (installDir: string, callbacks: PipelineCallbac log('Too few indexed notes found in native DB. Falling back to local ONNX Web Worker.'); } else { callbacks.onStatus('Clustering...'); - const adaptiveConfig = createAdaptiveConfig(nativeResult.dimension, validNotes.length, userMetric, userSeed); + const adaptiveConfig = createAdaptiveConfig( + nativeResult.dimension, + validNotes.length, + userMetric, + userSeed, + ); const results = benchmark(vectors, adaptiveConfig); // Post-process to extract tags/keywords for each cluster (keep parity with local pipeline) diff --git a/src/settings/registerSettings.ts b/src/settings/registerSettings.ts index a3a6551..4a45aa7 100644 --- a/src/settings/registerSettings.ts +++ b/src/settings/registerSettings.ts @@ -55,7 +55,9 @@ export async function runNativeCleanup(source: string, operationState: Operation await joplin.views.dialogs.showMessageBox(lastMessage); } } catch (err) { - await joplin.views.dialogs.showMessageBox(`Cleanup failed: ${err instanceof Error ? err.message : String(err)}`); + await joplin.views.dialogs.showMessageBox( + `Cleanup failed: ${err instanceof Error ? err.message : String(err)}`, + ); } finally { operationState.inProgress = false; } @@ -75,7 +77,8 @@ export async function registerPluginSettings(operationState: OperationState): Pr section: 'aiCategorization', public: true, label: 'Default Target Notebook', - description: 'Default parent notebook where newly categorized sub-notebooks will be created (leave empty for root).', + description: + 'Default parent notebook where newly categorized sub-notebooks will be created (leave empty for root).', }, 'categorization.changeLog': { value: '', @@ -99,7 +102,8 @@ export async function registerPluginSettings(operationState: OperationState): Pr section: 'aiCategorization', public: true, label: 'Undo Last Categorization', - description: 'Check this box and click Apply/OK to revert note movements and tags from the previous run.', + description: + 'Check this box and click Apply/OK to revert note movements and tags from the previous run.', }, 'categorization.cleanUpAction': { value: false, @@ -107,7 +111,8 @@ export async function registerPluginSettings(operationState: OperationState): Pr section: 'aiCategorization', public: true, label: 'Clean Up Empty Notebooks', - description: 'Check this box and click Apply/OK to check for and remove empty notebooks leftover after note moves.', + description: + 'Check this box and click Apply/OK to check for and remove empty notebooks leftover after note moves.', }, }); diff --git a/src/types/panel.ts b/src/types/panel.ts index 8f979cc..81e3f42 100644 --- a/src/types/panel.ts +++ b/src/types/panel.ts @@ -51,12 +51,7 @@ export type WebviewMessage = | { type: 'run' } | { type: 'poll' } | { type: 'getInitialState' } - | { - type: 'syncState'; - strategies: BenchmarkResult[]; - notes: PanelNote[]; - selectedStrategyIndex: number; - } + | { type: 'syncState'; strategies: BenchmarkResult[]; notes: PanelNote[]; selectedStrategyIndex: number } | { type: 'openNote'; noteId: string } | { type: 'getSettings' } | { type: 'updateSetting'; key: string; value: string } diff --git a/src/webview/context/AppStateContext.tsx b/src/webview/context/AppStateContext.tsx index f4ff612..86087a2 100644 --- a/src/webview/context/AppStateContext.tsx +++ b/src/webview/context/AppStateContext.tsx @@ -26,7 +26,6 @@ interface AppStateContextType { // settings states settings: { - metric: string; parentNotebook: string; changeLog: string; }; @@ -283,7 +282,8 @@ export const AppStateProvider: React.FC<{ children: React.ReactNode }> = ({ chil React.useEffect(() => { fetchSettings(); if (typeof webviewApi !== 'undefined') { - webviewApi.postMessage({ type: 'getInitialState' }) + webviewApi + .postMessage({ type: 'getInitialState' }) .then((initialState) => { if (initialState) { handlePollResponse(initialState); @@ -300,14 +300,16 @@ export const AppStateProvider: React.FC<{ children: React.ReactNode }> = ({ chil React.useEffect(() => { if (typeof webviewApi !== 'undefined' && strategies && strategies.length > 0) { - webviewApi.postMessage({ - type: 'syncState', - strategies, - notes, - selectedStrategyIndex, - }).catch((err) => { - console.error('syncState error:', err); - }); + webviewApi + .postMessage({ + type: 'syncState', + strategies, + notes, + selectedStrategyIndex, + }) + .catch((err) => { + console.error('syncState error:', err); + }); } }, [strategies, notes, selectedStrategyIndex]); diff --git a/src/webview/context/useApplyState.ts b/src/webview/context/useApplyState.ts index 112bb5e..f8b2a47 100644 --- a/src/webview/context/useApplyState.ts +++ b/src/webview/context/useApplyState.ts @@ -26,11 +26,7 @@ export function useApplyState(startPolling: () => void) { }, []); const applyChanges = React.useCallback( - async ( - options: ApplyOptions, - notes: PanelNote[], - currentStrategy: BenchmarkResult | undefined, - ) => { + async (options: ApplyOptions, notes: PanelNote[], currentStrategy: BenchmarkResult | undefined) => { if (!currentStrategy) { setApplyError('No active strategy selected.'); return; diff --git a/src/webview/pages/SettingsPage.tsx b/src/webview/pages/SettingsPage.tsx index 5f771c1..cbb3be4 100644 --- a/src/webview/pages/SettingsPage.tsx +++ b/src/webview/pages/SettingsPage.tsx @@ -8,11 +8,14 @@ export const SettingsPage: React.FC = () => {
Plugin Settings
- Settings are managed in Joplin's native Options window. - Go to Tools → Options → AI Categorization to configure the plugin. + Settings are managed in Joplin's native Options window. Go to{' '} + Tools → Options → AI Categorization to configure the plugin.
-
+
Active Configuration:
• Distance Metric: Cosine Similarity diff --git a/src/webview/panel.tsx b/src/webview/panel.tsx index 18de726..f08b14a 100644 --- a/src/webview/panel.tsx +++ b/src/webview/panel.tsx @@ -11,9 +11,7 @@ const AppContent: React.FC = () => {
{error &&
Error: {error}
} -
- {activeView === 'idle' ? : } -
+
{activeView === 'idle' ? : }
); }; diff --git a/test/pipeline/clustering/autoK.test.ts b/test/pipeline/clustering/autoK.test.ts index 7b86001..f6bdf93 100644 --- a/test/pipeline/clustering/autoK.test.ts +++ b/test/pipeline/clustering/autoK.test.ts @@ -90,19 +90,19 @@ describe('autoK computeKRange', () => { }); it('computes correct range for small-to-medium datasets (N < 20, uses N/2)', () => { - expect(computeKRange(6)).toEqual([2, 3]); // floor(6/2) = 3 - expect(computeKRange(8)).toEqual([2, 4]); // floor(8/2) = 4 - expect(computeKRange(9)).toEqual([2, 4]); // floor(9/2) = 4 + expect(computeKRange(6)).toEqual([2, 3]); // floor(6/2) = 3 + expect(computeKRange(8)).toEqual([2, 4]); // floor(8/2) = 4 + expect(computeKRange(9)).toEqual([2, 4]); // floor(9/2) = 4 expect(computeKRange(10)).toEqual([2, 5]); // floor(10/2) = 5 expect(computeKRange(12)).toEqual([2, 6]); // floor(12/2) = 6 expect(computeKRange(19)).toEqual([2, 9]); // floor(19/2) = 9 }); it('computes correct range for larger datasets (N >= 20, uses N/3)', () => { - expect(computeKRange(20)).toEqual([2, 6]); // floor(20/3) = 6 - expect(computeKRange(30)).toEqual([2, 10]); // floor(30/3) = 10 - expect(computeKRange(45)).toEqual([2, 15]); // floor(45/3) = 15, hits cap - expect(computeKRange(56)).toEqual([2, 15]); // floor(56/3) = 18, capped at 15 + expect(computeKRange(20)).toEqual([2, 6]); // floor(20/3) = 6 + expect(computeKRange(30)).toEqual([2, 10]); // floor(30/3) = 10 + expect(computeKRange(45)).toEqual([2, 15]); // floor(45/3) = 15, hits cap + expect(computeKRange(56)).toEqual([2, 15]); // floor(56/3) = 18, capped at 15 expect(computeKRange(100)).toEqual([2, 15]); // floor(100/3) = 33, capped at 15 expect(computeKRange(500)).toEqual([2, 15]); // capped at MAX_K_CAP=15 }); @@ -158,13 +158,25 @@ describe('autoK findOptimalK', () => { // should be within the 0.025 tolerance band and thus selected. const FOUR_CLUSTERS = [ // Cluster near [0, 0] - ...[0.1, 0.2, 0.0, 0.15, 0.05, 0.12, 0.08, 0.18].map((x, i) => [x, [0.2, 0.1, 0.0, 0.15, 0.05, 0.08, 0.12, 0.11][i]]), + ...[0.1, 0.2, 0.0, 0.15, 0.05, 0.12, 0.08, 0.18].map((x, i) => [ + x, + [0.2, 0.1, 0.0, 0.15, 0.05, 0.08, 0.12, 0.11][i], + ]), // Cluster near [10, 0] - ...[10.1, 10.2, 10.0, 10.15, 10.05, 10.12, 10.08, 10.18].map((x, i) => [x, [0.2, 0.1, 0.0, 0.15, 0.05, 0.08, 0.12, 0.11][i]]), + ...[10.1, 10.2, 10.0, 10.15, 10.05, 10.12, 10.08, 10.18].map((x, i) => [ + x, + [0.2, 0.1, 0.0, 0.15, 0.05, 0.08, 0.12, 0.11][i], + ]), // Cluster near [0, 10] - ...[0.1, 0.2, 0.0, 0.15, 0.05, 0.12, 0.08, 0.18].map((x, i) => [x, 10 + [0.2, 0.1, 0.0, 0.15, 0.05, 0.08, 0.12, 0.11][i]]), + ...[0.1, 0.2, 0.0, 0.15, 0.05, 0.12, 0.08, 0.18].map((x, i) => [ + x, + 10 + [0.2, 0.1, 0.0, 0.15, 0.05, 0.08, 0.12, 0.11][i], + ]), // Cluster near [10, 10] - ...[10.1, 10.2, 10.0, 10.15, 10.05, 10.12, 10.08, 10.18].map((x, i) => [x, 10 + [0.2, 0.1, 0.0, 0.15, 0.05, 0.08, 0.12, 0.11][i]]), + ...[10.1, 10.2, 10.0, 10.15, 10.05, 10.12, 10.08, 10.18].map((x, i) => [ + x, + 10 + [0.2, 0.1, 0.0, 0.15, 0.05, 0.08, 0.12, 0.11][i], + ]), ]; const result = findOptimalK(FOUR_CLUSTERS, 'kmeans', euclideanDistance, 42);