From aca0067b9c355022e4edd19e04ea8bea32089a65 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Thu, 9 Jul 2026 19:03:36 +0530 Subject: [PATCH 01/28] ANG-007: Markdown preprocessing & embedding pipeline --- api/Global.d.ts | 6 +- api/Joplin.d.ts | 15 +- api/JoplinAi.d.ts | 80 ++++++++ api/JoplinClipboard.d.ts | 18 +- api/JoplinContentScripts.d.ts | 4 +- api/JoplinData.d.ts | 5 +- api/JoplinFs.d.ts | 22 ++ api/JoplinImaging.d.ts | 10 +- api/JoplinSettings.d.ts | 2 +- api/JoplinViews.d.ts | 4 +- api/JoplinViewsDialogs.d.ts | 6 +- api/JoplinViewsEditor.d.ts | 7 +- api/JoplinViewsMenuItems.d.ts | 3 +- api/JoplinViewsMenus.d.ts | 3 +- api/JoplinViewsPanels.d.ts | 7 +- api/JoplinViewsToolbarButtons.d.ts | 3 +- api/JoplinWindow.d.ts | 9 +- api/JoplinWorkspace.d.ts | 7 +- api/noteListType.d.ts | 6 +- api/noteListType.ts | 10 +- api/types.ts | 155 +++++++++++--- package-lock.json | 14 +- package.json | 7 +- src/index.ts | 14 +- src/services/embeddings/Orchestrator.test.ts | 102 +++++++++ src/services/embeddings/Orchestrator.ts | 72 +++++++ .../embeddings/ProviderResolver.test.ts | 47 +++++ src/services/embeddings/ProviderResolver.ts | 26 +++ .../Providers/JoplinNativeProvider.ts | 116 +++++++++++ src/services/embeddings/Types.ts | 31 +++ src/services/graph/GraphBuilder.ts | 5 - src/tests/mocks/joplin.ts | 12 +- src/ui/graph-view.js | 20 +- src/ui/setup.js | 8 +- src/ui/styles/panel.css | 194 ++++++++++++++++++ src/ui/webview.ts | 11 - tsconfig.json | 3 +- 37 files changed, 967 insertions(+), 97 deletions(-) create mode 100644 api/JoplinAi.d.ts create mode 100644 api/JoplinFs.d.ts create mode 100644 src/services/embeddings/Orchestrator.test.ts create mode 100644 src/services/embeddings/Orchestrator.ts create mode 100644 src/services/embeddings/ProviderResolver.test.ts create mode 100644 src/services/embeddings/ProviderResolver.ts create mode 100644 src/services/embeddings/Providers/JoplinNativeProvider.ts create mode 100644 src/services/embeddings/Types.ts diff --git a/api/Global.d.ts b/api/Global.d.ts index 686ccf1..56bb162 100644 --- a/api/Global.d.ts +++ b/api/Global.d.ts @@ -1,5 +1,7 @@ import Plugin from '../Plugin'; import Joplin from './Joplin'; +import BasePlatformImplementation from '../BasePlatformImplementation'; +import type { Store } from 'redux'; /** * @ignore */ @@ -8,7 +10,7 @@ import Joplin from './Joplin'; */ export default class Global { private joplin_; - constructor(implementation: any, plugin: Plugin, store: any); + constructor(implementation: BasePlatformImplementation, plugin: Plugin, store: Store); get joplin(): Joplin; - get process(): any; + get process(): NodeJS.Process; } diff --git a/api/Joplin.d.ts b/api/Joplin.d.ts index 15c672c..ca1617e 100644 --- a/api/Joplin.d.ts +++ b/api/Joplin.d.ts @@ -12,6 +12,9 @@ import JoplinClipboard from './JoplinClipboard'; import JoplinWindow from './JoplinWindow'; import BasePlatformImplementation from '../BasePlatformImplementation'; import JoplinImaging from './JoplinImaging'; +import JoplinFs from './JoplinFs'; +import JoplinAi from './JoplinAi'; +import type { Store } from 'redux'; /** * This is the main entry point to the Joplin API. You can access various services using the provided accessors. * @@ -28,6 +31,7 @@ export default class Joplin { private data_; private plugins_; private imaging_; + private fs_; private workspace_; private filters_; private commands_; @@ -37,11 +41,13 @@ export default class Joplin { private contentScripts_; private clipboard_; private window_; + private ai_; private implementation_; - constructor(implementation: BasePlatformImplementation, plugin: Plugin, store: any); + constructor(implementation: BasePlatformImplementation, plugin: Plugin, store: Store); get data(): JoplinData; get clipboard(): JoplinClipboard; get imaging(): JoplinImaging; + get fs(): JoplinFs; get window(): JoplinWindow; get plugins(): JoplinPlugins; get workspace(): JoplinWorkspace; @@ -57,6 +63,13 @@ export default class Joplin { get views(): JoplinViews; get interop(): JoplinInterop; get settings(): JoplinSettings; + /** + * Access to AI features: chat completions and semantic search over the + * local embeddings index. See {@link JoplinAi}. + * + * desktop + */ + get ai(): JoplinAi; /** * It is not possible to bundle native packages with a plugin, because they * need to work cross-platforms. Instead access to certain useful native diff --git a/api/JoplinAi.d.ts b/api/JoplinAi.d.ts new file mode 100644 index 0000000..9e611b2 --- /dev/null +++ b/api/JoplinAi.d.ts @@ -0,0 +1,80 @@ +import { ChatMessage, ChatOptions, SearchOptions, SearchResult } from './types'; +/** + * Provides access to AI models configured by the user. The active provider + * (Joplin Cloud AI, OpenAI-compatible, or Anthropic) and the model are picked + * by the user in the Joplin settings — plugins inherit whichever is active. + * + * AI is disabled by default. The user must enable it in the settings, and + * separately grant permission to use a remote (cloud-hosted) provider before + * any plugin call will succeed. + * + * If the user is signed into Joplin Cloud, AI works zero-config — they only + * need to flip the master toggle on. + * + * desktop + */ +export default class JoplinAi { + /** + * Sends a chat completion request to the active AI provider and returns the + * assistant's text response. + * + * The active provider and model are controlled by the user in Settings → + * AI. Plugins should not assume any particular provider or model. + * + * This call throws when: + * + * - AI features are disabled (`AI features are disabled`). + * - The active provider is remote and the user has not allowed remote + * providers (`Remote AI access is not allowed`). + * - The provider is misconfigured, e.g. missing API key or model name + * (`*provider* has no API key configured`). + * - The provider returns an HTTP error (the message includes the status + * and any detail returned by the provider). + * + * Plugins should catch these errors and present a user-friendly message + * pointing the user at the Joplin settings. + * + * @example + * ```typescript + * const reply = await joplin.ai.chat([ + * { role: 'system', content: 'You are a concise assistant.' }, + * { role: 'user', content: 'Summarise this note: ...' }, + * ]); + * console.log(reply); + * ``` + */ + chat(messages: ChatMessage[], options?: ChatOptions): Promise; + /** + * Runs a semantic search against the locally-indexed embeddings and + * returns matching chunks ranked by similarity. + * + * The `query` is either plain text (which gets embedded internally) or + * `{ noteId }`, which reuses the note's already-indexed chunks as the + * query — useful for "find related notes" / tag suggestion / semantic + * graph use cases without spending another embedding pass. + * + * The `scope` restricts the search: `'all'` (default), `'note'`, + * `'folder'` (by folder id), or `'tag'` (by tag id). + * Trashed and conflict notes are excluded from results. + * + * The `relevance` preset controls how strict the match is: + * `'strict' | 'normal' | 'loose'`. Joplin owns the mapping from preset + * to model-specific (k, minScore) — plugins write against the preset + * and stay compatible when the bundled model changes. + * + * Throws when AI features are disabled or no embedding provider is + * active (e.g. ONNX failed to load on this platform). + * + * @example + * ```typescript + * const results = await joplin.ai.search({ + * query: { text: 'pizza dough hydration' }, + * relevance: 'normal', + * }); + * for (const r of results) { + * console.log(r.score, r.noteId, r.chunkText.slice(0, 80)); + * } + * ``` + */ + search(options: SearchOptions): Promise; +} diff --git a/api/JoplinClipboard.d.ts b/api/JoplinClipboard.d.ts index 6d2baa8..60c4c1c 100644 --- a/api/JoplinClipboard.d.ts +++ b/api/JoplinClipboard.d.ts @@ -1,8 +1,23 @@ import { ClipboardContent } from './types'; +interface ElectronClipboardLike { + readText(): string; + writeText(text: string): void; + readHTML(): string; + writeHTML(html: string): void; + readImage(): { + toDataURL(): string; + } | null; + writeImage(image: unknown): void; + availableFormats(): string[]; + write(data: Record): void; +} +interface ElectronNativeImageLike { + createFromDataURL(dataUrl: string): unknown; +} export default class JoplinClipboard { private electronClipboard_; private electronNativeImage_; - constructor(electronClipboard: any, electronNativeImage: any); + constructor(electronClipboard: ElectronClipboardLike, electronNativeImage: ElectronNativeImageLike); readText(): Promise; writeText(text: string): Promise; /** desktop */ @@ -43,3 +58,4 @@ export default class JoplinClipboard { */ write(content: ClipboardContent): Promise; } +export {}; diff --git a/api/JoplinContentScripts.d.ts b/api/JoplinContentScripts.d.ts index adf4e8d..d391ce0 100644 --- a/api/JoplinContentScripts.d.ts +++ b/api/JoplinContentScripts.d.ts @@ -1,4 +1,4 @@ -import Plugin from '../Plugin'; +import Plugin, { MessageListenerCallback } from '../Plugin'; import { ContentScriptType } from './types'; export default class JoplinContentScripts { private plugin; @@ -37,5 +37,5 @@ export default class JoplinContentScripts { * [postMessage * demo](https://github.com/laurent22/joplin/tree/dev/packages/app-cli/tests/support/plugins/post_messages) */ - onMessage(contentScriptId: string, callback: any): Promise; + onMessage(contentScriptId: string, callback: MessageListenerCallback): Promise; } diff --git a/api/JoplinData.d.ts b/api/JoplinData.d.ts index 026150a..e76db36 100644 --- a/api/JoplinData.d.ts +++ b/api/JoplinData.d.ts @@ -1,4 +1,5 @@ import { ModelType } from '../../../BaseModel'; +import { RequestFile } from '../../rest/Api'; import Plugin from '../Plugin'; import { Path } from './types'; /** @@ -45,8 +46,8 @@ export default class JoplinData { private serializeApiBody; private pathToString; get(path: Path, query?: any): Promise; - post(path: Path, query?: any, body?: any, files?: any[]): Promise; - put(path: Path, query?: any, body?: any, files?: any[]): Promise; + post(path: Path, query?: any, body?: any, files?: RequestFile[]): Promise; + put(path: Path, query?: any, body?: any, files?: RequestFile[]): Promise; delete(path: Path, query?: any): Promise; itemType(itemId: string): Promise; resourcePath(resourceId: string): Promise; diff --git a/api/JoplinFs.d.ts b/api/JoplinFs.d.ts new file mode 100644 index 0000000..238d5da --- /dev/null +++ b/api/JoplinFs.d.ts @@ -0,0 +1,22 @@ +export interface ArchiveEntry { + entryName: string; + name: string; +} +/** + * Provides file system utilities for plugins. + * + * desktop + */ +export default class JoplinFs { + /** + * Extracts an archive to the specified directory. Currently only ZIP files + * are supported. + * + * desktop + * + * @param sourcePath Path to the archive file to extract + * @param destinationPath Path to the directory where the contents should be extracted + * @returns List of entries extracted from the archive + */ + archiveExtract(sourcePath: string, destinationPath: string): Promise; +} diff --git a/api/JoplinImaging.d.ts b/api/JoplinImaging.d.ts index 8d878b5..7c5d469 100644 --- a/api/JoplinImaging.d.ts +++ b/api/JoplinImaging.d.ts @@ -1,3 +1,4 @@ +import { ResourceEntity } from '../../database/types'; import { Rectangle } from './types'; export interface CreateFromBufferOptions { width?: number; @@ -65,7 +66,10 @@ export default class JoplinImaging { createFromPdfResource(resourceId: string, options?: CreateFromPdfOptions): Promise; getPdfInfoFromPath(path: string): Promise; getPdfInfoFromResource(resourceId: string): Promise; - getSize(handle: Handle): Promise; + getSize(handle: Handle): Promise<{ + width: number; + height: number; + }>; resize(handle: Handle, options?: ResizeOptions): Promise; crop(handle: Handle, rectangle: Rectangle): Promise; toPngFile(handle: Handle, filePath: string): Promise; @@ -78,12 +82,12 @@ export default class JoplinImaging { * Creates a new Joplin resource from the image data. The image will be * first converted to a JPEG. */ - toJpgResource(handle: Handle, resourceProps: any, quality?: number): Promise; + toJpgResource(handle: Handle, resourceProps: Partial, quality?: number): Promise; /** * Creates a new Joplin resource from the image data. The image will be * first converted to a PNG. */ - toPngResource(handle: Handle, resourceProps: any): Promise; + toPngResource(handle: Handle, resourceProps: Partial): Promise; /** * Image data is not automatically deleted by Joplin so make sure you call * this method on the handle once you are done. diff --git a/api/JoplinSettings.d.ts b/api/JoplinSettings.d.ts index 13cdca2..a3aa324 100644 --- a/api/JoplinSettings.d.ts +++ b/api/JoplinSettings.d.ts @@ -40,7 +40,7 @@ export default class JoplinSettings { /** * Gets setting values (only applies to setting you registered from your plugin) */ - values(keys: string[] | string): Promise>; + values(keys: string[] | string): Promise>; /** * Gets a setting value (only applies to setting you registered from your plugin). * diff --git a/api/JoplinViews.d.ts b/api/JoplinViews.d.ts index 364e82a..237286b 100644 --- a/api/JoplinViews.d.ts +++ b/api/JoplinViews.d.ts @@ -1,4 +1,6 @@ +import { JoplinViews as JoplinViewsImplementation } from '../BasePlatformImplementation'; import Plugin from '../Plugin'; +import { PluginStore } from '../ViewController'; import JoplinViewsDialogs from './JoplinViewsDialogs'; import JoplinViewsMenuItems from './JoplinViewsMenuItems'; import JoplinViewsMenus from './JoplinViewsMenus'; @@ -32,7 +34,7 @@ export default class JoplinViews { private editors_; private noteList_; private implementation_; - constructor(implementation: any, plugin: Plugin, store: any); + constructor(implementation: JoplinViewsImplementation, plugin: Plugin, store: PluginStore); get dialogs(): JoplinViewsDialogs; get panels(): JoplinViewsPanels; get editors(): JoplinViewsEditors; diff --git a/api/JoplinViewsDialogs.d.ts b/api/JoplinViewsDialogs.d.ts index 55db518..a331545 100644 --- a/api/JoplinViewsDialogs.d.ts +++ b/api/JoplinViewsDialogs.d.ts @@ -1,5 +1,7 @@ import Plugin from '../Plugin'; import { ButtonSpec, ViewHandle, DialogResult, Toast } from './types'; +import { JoplinViewsDialogs as JoplinViewsDialogsImplementation, ShowOpenDialogOptions } from '../BasePlatformImplementation'; +import { PluginStore } from '../ViewController'; /** * Allows creating and managing dialogs. A dialog is modal window that * contains a webview and a row of buttons. You can update the @@ -33,7 +35,7 @@ export default class JoplinViewsDialogs { private store; private plugin; private implementation_; - constructor(implementation: any, plugin: Plugin, store: any); + constructor(implementation: JoplinViewsDialogsImplementation, plugin: Plugin, store: PluginStore); private controller; /** * Creates a new dialog @@ -54,7 +56,7 @@ export default class JoplinViewsDialogs { * * desktop */ - showOpenDialog(options: any): Promise; + showOpenDialog(options: ShowOpenDialogOptions): Promise; /** * Sets the dialog HTML content */ diff --git a/api/JoplinViewsEditor.d.ts b/api/JoplinViewsEditor.d.ts index 512c596..72bd953 100644 --- a/api/JoplinViewsEditor.d.ts +++ b/api/JoplinViewsEditor.d.ts @@ -1,4 +1,5 @@ -import Plugin from '../Plugin'; +import Plugin, { MessageListenerCallback } from '../Plugin'; +import { PluginStore } from '../ViewController'; import { ActivationCheckCallback, ViewHandle, UpdateCallback, EditorPluginCallbacks } from './types'; interface SaveNoteOptions { /** @@ -55,7 +56,7 @@ export default class JoplinViewsEditors { private plugin; private activationCheckHandlers_; private unhandledActivationCheck_; - constructor(plugin: Plugin, store: any); + constructor(plugin: Plugin, store: PluginStore); private controller; /** * Registers a new editor plugin. Joplin will call the provided callback to create new editor views @@ -79,7 +80,7 @@ export default class JoplinViewsEditors { /** * See [[JoplinViewPanels]] */ - onMessage(handle: ViewHandle, callback: Function): Promise; + onMessage(handle: ViewHandle, callback: MessageListenerCallback): Promise; /** * Saves the content of the editor, without calling `onUpdate` for editors in the same window. */ diff --git a/api/JoplinViewsMenuItems.d.ts b/api/JoplinViewsMenuItems.d.ts index 5e236b1..d8b46f5 100644 --- a/api/JoplinViewsMenuItems.d.ts +++ b/api/JoplinViewsMenuItems.d.ts @@ -1,5 +1,6 @@ import { CreateMenuItemOptions, MenuItemLocation } from './types'; import Plugin from '../Plugin'; +import { PluginStore } from '../ViewController'; /** * Allows creating and managing menu items. * @@ -10,7 +11,7 @@ import Plugin from '../Plugin'; export default class JoplinViewsMenuItems { private store; private plugin; - constructor(plugin: Plugin, store: any); + constructor(plugin: Plugin, store: PluginStore); /** * Creates a new menu item and associate it with the given command. You can specify under which menu the item should appear using the `location` parameter. */ diff --git a/api/JoplinViewsMenus.d.ts b/api/JoplinViewsMenus.d.ts index 474830d..67f6d6d 100644 --- a/api/JoplinViewsMenus.d.ts +++ b/api/JoplinViewsMenus.d.ts @@ -1,5 +1,6 @@ import { MenuItem, MenuItemLocation } from './types'; import Plugin from '../Plugin'; +import { PluginStore } from '../ViewController'; /** * Allows creating menus. * @@ -10,7 +11,7 @@ import Plugin from '../Plugin'; export default class JoplinViewsMenus { private store; private plugin; - constructor(plugin: Plugin, store: any); + constructor(plugin: Plugin, store: PluginStore); private registerCommandAccelerators; /** * Creates a new menu from the provided menu items and place it at the given location. As of now, it is only possible to place the diff --git a/api/JoplinViewsPanels.d.ts b/api/JoplinViewsPanels.d.ts index 881dbb0..73259da 100644 --- a/api/JoplinViewsPanels.d.ts +++ b/api/JoplinViewsPanels.d.ts @@ -1,4 +1,5 @@ -import Plugin from '../Plugin'; +import Plugin, { MessageListenerCallback } from '../Plugin'; +import { PluginStore } from '../ViewController'; import { ViewHandle } from './types'; /** * Allows creating and managing view panels. View panels allow displaying any HTML @@ -17,7 +18,7 @@ import { ViewHandle } from './types'; export default class JoplinViewsPanels { private store; private plugin; - constructor(plugin: Plugin, store: any); + constructor(plugin: Plugin, store: PluginStore); private controller; /** * Creates a new panel @@ -50,7 +51,7 @@ export default class JoplinViewsPanels { * demo](https://github.com/laurent22/joplin/tree/dev/packages/app-cli/tests/support/plugins/post_messages) for more details. * */ - onMessage(handle: ViewHandle, callback: Function): Promise; + onMessage(handle: ViewHandle, callback: MessageListenerCallback): Promise; /** * Sends a message to the webview. * diff --git a/api/JoplinViewsToolbarButtons.d.ts b/api/JoplinViewsToolbarButtons.d.ts index ba17c83..c1d12c2 100644 --- a/api/JoplinViewsToolbarButtons.d.ts +++ b/api/JoplinViewsToolbarButtons.d.ts @@ -1,5 +1,6 @@ import { ToolbarButtonLocation } from './types'; import Plugin from '../Plugin'; +import { PluginStore } from '../ViewController'; /** * Allows creating and managing toolbar buttons. * @@ -8,7 +9,7 @@ import Plugin from '../Plugin'; export default class JoplinViewsToolbarButtons { private store; private plugin; - constructor(plugin: Plugin, store: any); + constructor(plugin: Plugin, store: PluginStore); /** * Creates a new toolbar button and associate it with the given command. */ diff --git a/api/JoplinWindow.d.ts b/api/JoplinWindow.d.ts index 4cbdc64..ed9b3cb 100644 --- a/api/JoplinWindow.d.ts +++ b/api/JoplinWindow.d.ts @@ -1,7 +1,13 @@ import Plugin from '../Plugin'; +type DispatchStore = { + dispatch: (action: { + type: string; + [k: string]: unknown; + }) => void; +}; export default class JoplinWindow { private store_; - constructor(_plugin: Plugin, store: any); + constructor(_plugin: Plugin, store: DispatchStore); /** * Loads a chrome CSS file. It will apply to the window UI elements, except * for the note viewer. It is the same as the "Custom stylesheet for @@ -21,3 +27,4 @@ export default class JoplinWindow { */ loadNoteCssFile(filePath: string): Promise; } +export {}; diff --git a/api/JoplinWorkspace.d.ts b/api/JoplinWorkspace.d.ts index 9799f6f..e89f7e6 100644 --- a/api/JoplinWorkspace.d.ts +++ b/api/JoplinWorkspace.d.ts @@ -1,5 +1,6 @@ import Plugin from '../Plugin'; -import { FolderEntity } from '../../database/types'; +import { PluginStore } from '../ViewController'; +import { FolderEntity, NoteEntity } from '../../database/types'; import { Disposable, EditContextMenuFilterObject, FilterHandler } from './types'; declare enum ItemChangeEventType { Create = 1, @@ -40,7 +41,7 @@ type ResourceChangeHandler = WorkspaceEventHandler; export default class JoplinWorkspace { private store; private plugin; - constructor(plugin: Plugin, store: any); + constructor(plugin: Plugin, store: PluginStore); /** * Called when a new note or notes are selected. */ @@ -83,7 +84,7 @@ export default class JoplinWorkspace { * * On desktop, this returns the selected note in the focused window. */ - selectedNote(): Promise; + selectedNote(): Promise; /** * Gets the currently selected folder. In some cases, for example during * search or when viewing a tag, no folder is actually selected in the user diff --git a/api/noteListType.d.ts b/api/noteListType.d.ts index 2fc14ee..7862a63 100644 --- a/api/noteListType.d.ts +++ b/api/noteListType.d.ts @@ -1,5 +1,5 @@ import { Size } from './types'; -type ListRendererDatabaseDependency = 'folder.created_time' | 'folder.deleted_time' | 'folder.encryption_applied' | 'folder.encryption_cipher_text' | 'folder.icon' | 'folder.id' | 'folder.is_shared' | 'folder.master_key_id' | 'folder.parent_id' | 'folder.share_id' | 'folder.title' | 'folder.updated_time' | 'folder.user_created_time' | 'folder.user_data' | 'folder.user_updated_time' | 'folder.type_' | 'note.altitude' | 'note.application_data' | 'note.author' | 'note.body' | 'note.conflict_original_id' | 'note.created_time' | 'note.deleted_time' | 'note.encryption_applied' | 'note.encryption_cipher_text' | 'note.id' | 'note.is_conflict' | 'note.is_shared' | 'note.is_todo' | 'note.latitude' | 'note.longitude' | 'note.markup_language' | 'note.master_key_id' | 'note.order' | 'note.parent_id' | 'note.share_id' | 'note.source' | 'note.source_application' | 'note.source_url' | 'note.title' | 'note.todo_completed' | 'note.todo_due' | 'note.updated_time' | 'note.user_created_time' | 'note.user_data' | 'note.user_updated_time' | 'note.type_'; +type ListRendererDatabaseDependency = 'folder.created_time' | 'folder.deleted_time' | 'folder.encryption_applied' | 'folder.encryption_cipher_text' | 'folder.icon' | 'folder.id' | 'folder.is_shared' | 'folder.master_key_id' | 'folder.parent_id' | 'folder.share_id' | 'folder.title' | 'folder.updated_time' | 'folder.user_created_time' | 'folder.user_data' | 'folder.user_updated_time' | 'folder.type_' | 'note.altitude' | 'note.application_data' | 'note.author' | 'note.body' | 'note.conflict_original_id' | 'note.created_time' | 'note.deleted_time' | 'note.encryption_applied' | 'note.encryption_cipher_text' | 'note.extracted_resource_ids' | 'note.id' | 'note.is_conflict' | 'note.is_locked' | 'note.is_shared' | 'note.is_todo' | 'note.latitude' | 'note.longitude' | 'note.markup_language' | 'note.master_key_id' | 'note.order' | 'note.parent_id' | 'note.share_id' | 'note.source' | 'note.source_application' | 'note.source_url' | 'note.title' | 'note.todo_completed' | 'note.todo_due' | 'note.updated_time' | 'note.user_created_time' | 'note.user_data' | 'note.user_updated_time' | 'note.type_'; export declare enum ItemFlow { TopToBottom = "topToBottom", LeftToRight = "leftToRight" @@ -30,9 +30,9 @@ export type OnClickHandler = (event: OnClickEvent) => Promise; * The `item.*` properties are specific to the rendered item. The most important being * `item.selected`, which you can use to display the selected note in a different way. */ -export type ListRendererDependency = ListRendererDatabaseDependency | 'item.index' | 'item.selected' | 'item.size.height' | 'item.size.width' | 'note.folder.title' | 'note.isWatched' | 'note.tags' | 'note.todoStatusText' | 'note.titleHtml'; +export type ListRendererDependency = ListRendererDatabaseDependency | 'item.index' | 'item.selected' | 'item.size.height' | 'item.size.width' | 'note.checkboxes' | 'note.folder.title' | 'note.isWatched' | 'note.tags' | 'note.todoStatusText' | 'note.titleHtml'; export type ListRendererItemValueTemplates = Record; -export declare const columnNames: readonly ["note.folder.title", "note.is_todo", "note.latitude", "note.longitude", "note.source_url", "note.tags", "note.title", "note.todo_completed", "note.todo_due", "note.user_created_time", "note.user_updated_time"]; +export declare const columnNames: readonly ["note.checkboxes", "note.folder.title", "note.is_todo", "note.latitude", "note.longitude", "note.source_url", "note.tags", "note.title", "note.todo_completed", "note.todo_due", "note.user_created_time", "note.user_updated_time"]; export type ColumnName = typeof columnNames[number]; export interface ListRenderer { /** diff --git a/api/noteListType.ts b/api/noteListType.ts index ad00453..cf09294 100644 --- a/api/noteListType.ts +++ b/api/noteListType.ts @@ -3,7 +3,7 @@ import { Size } from './types'; // AUTO-GENERATED by generate-database-type -type ListRendererDatabaseDependency = 'folder.created_time' | 'folder.deleted_time' | 'folder.encryption_applied' | 'folder.encryption_cipher_text' | 'folder.icon' | 'folder.id' | 'folder.is_shared' | 'folder.master_key_id' | 'folder.parent_id' | 'folder.share_id' | 'folder.title' | 'folder.updated_time' | 'folder.user_created_time' | 'folder.user_data' | 'folder.user_updated_time' | 'folder.type_' | 'note.altitude' | 'note.application_data' | 'note.author' | 'note.body' | 'note.conflict_original_id' | 'note.created_time' | 'note.deleted_time' | 'note.encryption_applied' | 'note.encryption_cipher_text' | 'note.id' | 'note.is_conflict' | 'note.is_shared' | 'note.is_todo' | 'note.latitude' | 'note.longitude' | 'note.markup_language' | 'note.master_key_id' | 'note.order' | 'note.parent_id' | 'note.share_id' | 'note.source' | 'note.source_application' | 'note.source_url' | 'note.title' | 'note.todo_completed' | 'note.todo_due' | 'note.updated_time' | 'note.user_created_time' | 'note.user_data' | 'note.user_updated_time' | 'note.type_'; +type ListRendererDatabaseDependency = 'folder.created_time' | 'folder.deleted_time' | 'folder.encryption_applied' | 'folder.encryption_cipher_text' | 'folder.icon' | 'folder.id' | 'folder.is_shared' | 'folder.master_key_id' | 'folder.parent_id' | 'folder.share_id' | 'folder.title' | 'folder.updated_time' | 'folder.user_created_time' | 'folder.user_data' | 'folder.user_updated_time' | 'folder.type_' | 'note.altitude' | 'note.application_data' | 'note.author' | 'note.body' | 'note.conflict_original_id' | 'note.created_time' | 'note.deleted_time' | 'note.encryption_applied' | 'note.encryption_cipher_text' | 'note.extracted_resource_ids' | 'note.id' | 'note.is_conflict' | 'note.is_locked' | 'note.is_shared' | 'note.is_todo' | 'note.latitude' | 'note.longitude' | 'note.markup_language' | 'note.master_key_id' | 'note.order' | 'note.parent_id' | 'note.share_id' | 'note.source' | 'note.source_application' | 'note.source_url' | 'note.title' | 'note.todo_completed' | 'note.todo_due' | 'note.updated_time' | 'note.user_created_time' | 'note.user_data' | 'note.user_updated_time' | 'note.type_'; // AUTO-GENERATED by generate-database-type export enum ItemFlow { @@ -11,12 +11,12 @@ export enum ItemFlow { LeftToRight = 'leftToRight', } -// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied +// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Plugin-API output map; values are heterogeneous (HTML strings, formatted numbers, booleans) and indexed dynamically per-plugin export type RenderNoteView = Record; export interface OnChangeEvent { elementId: string; - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Plugin API: value depends on the input element type value: any; noteId: string; } @@ -25,7 +25,7 @@ export interface OnClickEvent { elementId: string; } -// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied +// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Plugin-API callback; props is a per-renderer subset of the note shape declared via itemProps, indexed dynamically export type OnRenderNoteHandler = (props: any)=> Promise; export type OnChangeHandler = (event: OnChangeEvent)=> Promise; export type OnClickHandler = (event: OnClickEvent)=> Promise; @@ -50,6 +50,7 @@ export type ListRendererDependency = 'item.selected' | 'item.size.height' | 'item.size.width' | + 'note.checkboxes' | 'note.folder.title' | 'note.isWatched' | 'note.tags' | @@ -59,6 +60,7 @@ export type ListRendererDependency = export type ListRendererItemValueTemplates = Record; export const columnNames = [ + 'note.checkboxes', 'note.folder.title', 'note.is_todo', 'note.latitude', diff --git a/api/types.ts b/api/types.ts index 3911a38..58c1614 100644 --- a/api/types.ts +++ b/api/types.ts @@ -26,7 +26,7 @@ export interface Command { /** * Code to be ran when the command is executed. It may return a result. */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Plugin commands accept arbitrary args and return arbitrary results; this is part of the public plugin API execute(...args: any[]): Promise; /** @@ -116,13 +116,13 @@ export interface ExportModule { /** * Called when an item needs to be processed. An "item" can be any Joplin object, such as a note, a folder, a notebook, etc. */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Plugin API: item type depends on itemType (NoteEntity, FolderEntity, ResourceEntity, etc.); plugin authors discriminate at use site onProcessItem(context: ExportContext, itemType: number, item: any): Promise; /** * Called when a resource file needs to be exported. */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- See onProcessItem; resource here is a ResourceEntity but the plugin API keeps it loosely typed onProcessResource(context: ExportContext, resource: any, filePath: string): Promise; /** @@ -186,13 +186,13 @@ export interface ExportContext { /** * You can attach your own custom data using this property - it will then be passed to each event handler, allowing you to keep state from one event to the next. */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Plugin API: userData is arbitrary per-plugin state userData?: any; } export interface ImportContext { sourcePath: string; - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Plugin API: import options are arbitrary per-importer options: any; warnings: string[]; } @@ -202,7 +202,7 @@ export interface ImportContext { // ================================================================= export interface Script { - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Plugin API: event payload shape depends on the host context onStart?(event: any): Promise; } @@ -308,7 +308,7 @@ export interface MenuItem { * Arguments that should be passed to the command. They will be as rest * parameters. */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Plugin API: command args depend on the command commandArgs?: any[]; /** @@ -362,13 +362,13 @@ export type ViewHandle = string; export interface EditorCommand { name: string; - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Plugin API: command value depends on the command value?: any; } export interface DialogResult { id: ButtonId; - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Plugin API: form data shape depends on the dialog formData?: any; } @@ -434,8 +434,30 @@ export interface EditorPluginCallbacks { export type VisibleHandler = ()=> Promise; +/** + * Identifies the type of element that was right-clicked in the editor context menu. + */ +export enum ContextMenuItemType { + None = '', + Image = 'image', + Resource = 'resource', + Text = 'text', + Link = 'link', + NoteLink = 'noteLink', +} + export interface EditContextMenuFilterObject { items: MenuItem[]; + /** + * Context about what was right-clicked. Plugins should use this instead of + * checking the editor cursor position, as the cursor may not reflect the + * actual click location. + */ + context?: { + resourceId?: string; + itemType?: ContextMenuItemType; + textToCopy?: string; + }; } export interface EditorActivationCheckFilterObject { @@ -497,7 +519,7 @@ export enum SettingStorage { // Redefine a simplified interface to mask internal details // and to remove function calls as they would have to be async. export interface SettingItem { - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Setting values are heterogeneous per setting (string/number/bool/Record/Array); plugin authors narrow at use site value: any; type: SettingItemType; @@ -534,8 +556,7 @@ export interface SettingItem { * This property is required when `isEnum` is `true`. In which case, it * should contain a map of value => label. */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied - options?: Record; + options?: Record; /** * Reserved property. Not used at the moment. @@ -616,7 +637,7 @@ export interface ClipboardContent { // Content Script types // ================================================================= -// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied +// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Plugin API: messages between content scripts and plugins are arbitrary serialisable data export type PostMessageHandler = (message: any)=> Promise; /** @@ -640,38 +661,38 @@ export interface ContentScriptContext { } export interface ContentScriptModuleLoadedEvent { - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Plugin API: userData is arbitrary per-plugin state userData?: any; } export interface ContentScriptModule { onLoaded?: (event: ContentScriptModuleLoadedEvent)=> void; - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Plugin entry point returns a plugin-specific module (markdown-it plugin, CodeMirror plugin, etc.); shape varies per content script type plugin: ()=> any; assets?: ()=> void; } export interface MarkdownItContentScriptModule extends Omit { - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- markdown-it and options are external library types not imported here; plugin authors annotate concretely plugin: (markdownIt: any, options: any)=> any; } -// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied +// eslint-disable-next-line @typescript-eslint/no-explicit-any -- CodeMirror command callbacks accept and return arbitrary values; matches CM6 Command type type EditorCommandCallback = (...args: any[])=> any; export interface CodeMirrorControl { /** Points to a CodeMirror 6 EditorView instance. */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- CM6 EditorView is an external library type not imported here editor: any; - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- CM6 module namespace; types come from the external library cm6: any; /** `extension` should be a [CodeMirror 6 extension](https://codemirror.net/docs/ref/#state.Extension). */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- CM6 Extension type comes from the external library addExtension(extension: any|any[]): void; supportsCommand(name: string): boolean; - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- See EditorCommandCallback execCommand(name: string, ...args: any[]): any; registerCommand(name: string, callback: EditorCommandCallback): void; @@ -685,13 +706,13 @@ export interface CodeMirrorControl { * * Using `autocompletion({ override: [ ... ]})` causes errors when done by multiple plugins. */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- CM6 CompletionSource and Extension types come from the external library completionSource(completionSource: any): any; /** * Creates an extension that enables or disables [`languageData`-based autocompletion](https://codemirror.net/docs/ref/#autocomplete.autocompletion^config.override). */ - // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied + // eslint-disable-next-line @typescript-eslint/no-explicit-any -- See completionSource above enableLanguageDataAutocomplete: { of: (enabled: boolean)=> any }; /** @@ -938,3 +959,91 @@ export enum ContentScriptType { */ CodeMirrorPlugin = 'codeMirrorPlugin', } + +// ================================================================= +// AI API types +// ================================================================= + +/** + * Role of a chat message. `system` messages set the assistant's behaviour, + * `user` messages come from the end user, and `assistant` messages are model + * responses fed back as conversation history. + */ +export type ChatMessageRole = 'system' | 'user' | 'assistant'; + +/** + * A single message in a chat conversation. + */ +export interface ChatMessage { + role: ChatMessageRole; + content: string; +} + +/** + * Optional parameters for a chat call. The active model and provider are + * controlled by the user in the Joplin settings — plugins cannot pick a model. + */ +export interface ChatOptions { + /** Sampling temperature, typically between 0 and 1. Provider default if omitted. */ + temperature?: number; + /** Maximum number of tokens to generate. Provider default if omitted. */ + maxTokens?: number; +} + +/** + * Relevance preset for semantic search. Maps internally to model-specific + * `(k, minScore)` tuning — the preset is the public contract so plugins keep + * working when the bundled embedding model changes. + */ +export type SearchRelevance = 'strict' | 'normal' | 'loose'; + +/** + * Where to look for matches. + * + * - `all`: every indexed note (default). + * - `note`: a single note (rarely useful directly — mainly an internal + * building block). + * - `folder`: all notes in the given folder (a "notebook" in the UI). + * - `tag`: all notes tagged with the given tag. + * + * Trashed and conflict notes are always excluded. + */ +export type SearchScope = + | { type: 'all' } + | { type: 'note'; noteId: string } + | { type: 'folder'; folderId: string } + | { type: 'tag'; tagId: string }; + +/** + * What to search for: free text (embedded internally), or an existing note + * whose stored chunks are reused as the query — useful for "related notes", + * tag suggestions, and graph-style use cases without a second embedding pass. + */ +export type SearchQuery = + | { text: string } + | { noteId: string }; + +/** + * Parameters for {@link JoplinAi.search}. + */ +export interface SearchOptions { + query: SearchQuery; + scope?: SearchScope; + relevance?: SearchRelevance; +} + +/** + * A single hit from {@link JoplinAi.search}. + */ +export interface SearchResult { + noteId: string; + chunkIndex: number; + chunkText: string; + /** + * Cosine similarity in `[0, 1]`. Higher means more similar. Plugins should + * use this for ranking but not as an absolute threshold — that's what the + * `relevance` preset is for. + */ + score: number; +} + diff --git a/package-lock.json b/package-lock.json index b6b66d4..f2a3a72 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,11 +10,12 @@ "license": "MIT", "dependencies": { "cytoscape": "^3.34.0", - "cytoscape-fcose": "^2.2.0" + "cytoscape-fcose": "^2.2.0", + "cytoscape-svg": "^0.4.0" }, "devDependencies": { "@types/jest": "^29.5.14", - "@types/node": "^18.19.130", + "@types/node": "^18.7.13", "chalk": "^4.1.0", "copy-webpack-plugin": "^11.0.0", "fs-extra": "^10.1.0", @@ -2188,6 +2189,15 @@ "cytoscape": "^3.2.0" } }, + "node_modules/cytoscape-svg": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/cytoscape-svg/-/cytoscape-svg-0.4.0.tgz", + "integrity": "sha512-omqIzfPd1Vy9mk6lHTiR2wTbjxELxb9GXSQ2pE6W+GwAe/6/yvOUQ2h5ApFf2QhCBnpMwLkCTq5DZXxBCgUpDw==", + "license": "GNU GPLv3", + "peerDependencies": { + "cytoscape": "^3.2.0" + } + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", diff --git a/package.json b/package.json index 2331564..a9893d3 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ ], "devDependencies": { "@types/jest": "^29.5.14", - "@types/node": "^18.19.130", + "@types/node": "^18.7.13", "chalk": "^4.1.0", "copy-webpack-plugin": "^11.0.0", "fs-extra": "^10.1.0", @@ -38,6 +38,7 @@ }, "dependencies": { "cytoscape": "^3.34.0", - "cytoscape-fcose": "^2.2.0" + "cytoscape-fcose": "^2.2.0", + "cytoscape-svg": "^0.4.0" } -} +} \ No newline at end of file diff --git a/src/index.ts b/src/index.ts index af8d9da..d6a99c0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,11 +8,16 @@ import { GraphBuilder } from './services/graph/GraphBuilder'; const SHOW_NOTE_GRAPH_COMMAND = 'showNoteGraph'; const SHOW_NOTE_GRAPH_MENU_ITEM = 'showNoteGraphMenuItem'; +const SETTINGS_SECTION = 'noteGraphSection'; + +const registerSettings = async (): Promise => { + await joplin.settings.registerSection(SETTINGS_SECTION, { + label: 'Note Graph', + iconName: 'fas fa-project-diagram', + description: 'Uses Joplin\'s built-in AI to discover connections between your notes. Enable AI in Settings → AI.', + }); +}; -/** - * Loads all notes from the Joplin API and enriches them with links and tags. - * @returns enriched notes ready for graph building. - */ export const loadNotes = async (): Promise => { const noteRepository = new NoteRepository(); const { notes } = await noteRepository.getAllNotes(); @@ -54,6 +59,7 @@ const registerMenuItems = async (): Promise => { joplin.plugins.register({ onStart: async function () { console.info('Note Graph plugin started.'); + await registerSettings(); await initializeAiNoteGraphPanel(); await registerCommands(); await registerMenuItems(); diff --git a/src/services/embeddings/Orchestrator.test.ts b/src/services/embeddings/Orchestrator.test.ts new file mode 100644 index 0000000..5e0587e --- /dev/null +++ b/src/services/embeddings/Orchestrator.test.ts @@ -0,0 +1,102 @@ +import { EmbeddingOrchestrator } from './Orchestrator'; +import { Note } from '../../data/Types'; + +function makeNote(id: string, title: string, body: string): Note { + return { + id, + parent_id: 'p1', + title, + body, + created_time: 0, + updated_time: 0, + }; +} + +describe('EmbeddingOrchestrator', () => { + let orchestrator: EmbeddingOrchestrator; + + beforeEach(() => { + orchestrator = new EmbeddingOrchestrator(); + }); + + describe('embedNotes', () => { + it('returns empty result for empty notes array', async () => { + const result = await orchestrator.embedNotes([]); + expect(result.embeddedNotes).toEqual([]); + expect(result.errors).toEqual([]); + }); + + it('returns error when no provider is set', async () => { + const result = await orchestrator.embedNotes([makeNote('n1', 'T1', 'Body')]); + expect(result.errors).toHaveLength(1); + expect(result.errors[0].error).toContain('No provider'); + }); + + it('maps fetched vectors to embedded notes', async () => { + const mockVectors = new Map(); + mockVectors.set('n1', [0.1, 0.2, 0.3]); + mockVectors.set('n2', [0.4, 0.5, 0.6]); + + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'test-model', + fetchVectorsByNoteIds: jest.fn().mockResolvedValue(mockVectors), + }); + + const notes = [makeNote('n1', 'Title 1', 'Body 1'), makeNote('n2', 'Title 2', 'Body 2')]; + const result = await orchestrator.embedNotes(notes); + + expect(result.embeddedNotes).toHaveLength(2); + expect(result.errors).toHaveLength(0); + expect(result.embeddedNotes[0].embedding).toEqual([0.1, 0.2, 0.3]); + expect(result.embeddedNotes[1].embedding).toEqual([0.4, 0.5, 0.6]); + }); + + it('reports errors for notes not found in index', async () => { + const mockVectors = new Map(); + mockVectors.set('n1', [0.1, 0.2, 0.3]); + + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'test-model', + fetchVectorsByNoteIds: jest.fn().mockResolvedValue(mockVectors), + }); + + const result = await orchestrator.embedNotes([ + makeNote('n1', 'T1', 'B1'), + makeNote('n2', 'T2', 'B2'), + ]); + + expect(result.embeddedNotes).toHaveLength(1); + expect(result.errors).toHaveLength(1); + expect(result.errors[0].noteId).toBe('n2'); + }); + + it('returns empty when cancelled', async () => { + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'test-model', + fetchVectorsByNoteIds: jest.fn().mockImplementation(async () => { + orchestrator.cancel(); + return new Map(); + }), + }); + + const result = await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1')]); + expect(result.embeddedNotes).toEqual([]); + }); + + it('catches provider errors and marks all notes', async () => { + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'test-model', + fetchVectorsByNoteIds: jest.fn().mockRejectedValue(new Error('API failure')), + }); + + const result = await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1')]); + expect(result.embeddedNotes).toHaveLength(0); + expect(result.errors).toHaveLength(1); + expect(result.errors[0].error).toBe('API failure'); + }); + }); +}); diff --git a/src/services/embeddings/Orchestrator.ts b/src/services/embeddings/Orchestrator.ts new file mode 100644 index 0000000..64332e3 --- /dev/null +++ b/src/services/embeddings/Orchestrator.ts @@ -0,0 +1,72 @@ +import { Note } from '../../data/Types'; +import { + EmbeddingProvider, + EmbeddedNote, + EmbeddingResult, + BatchProgress, +} from './Types'; + +export class EmbeddingOrchestrator { + private provider: EmbeddingProvider | null = null; + private cancelled: boolean = false; + private onProgress: ((progress: BatchProgress) => void) | null = null; + + public setProvider(provider: EmbeddingProvider): void { + this.provider = provider; + } + + public setOnProgress(callback: (progress: BatchProgress) => void): void { + this.onProgress = callback; + } + + public cancel(): void { + this.cancelled = true; + } + + public async embedNotes(notes: Note[]): Promise { + const embeddedNotes: EmbeddedNote[] = []; + const errors: Array<{ noteId: string; error: string }> = []; + + if (!notes || notes.length === 0) { + return { embeddedNotes, errors }; + } + + if (!this.provider) { + return { embeddedNotes, errors: notes.map(function (n) { return { noteId: n.id, error: 'No provider configured' }; }) }; + } + + try { + this.reportProgress(0, notes.length, 'embedding'); + + const noteIds = notes.map(function (n) { return n.id; }); + const vectorsByNoteId = await this.provider.fetchVectorsByNoteIds(noteIds); + + for (let i = 0; i < notes.length; i++) { + if (this.cancelled) break; + const note = notes[i]; + const vector = vectorsByNoteId.get(note.id); + if (vector) { + embeddedNotes.push({ note: note, embedding: vector }); + } else { + errors.push({ noteId: note.id, error: 'Note not yet indexed by Joplin AI. Wait for indexing to complete.' }); + } + this.reportProgress(i + 1, notes.length, 'embedding'); + } + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + for (let n = 0; n < notes.length; n++) { + if (!embeddedNotes.some(function (en) { return en.note.id === notes[n].id; })) { + errors.push({ noteId: notes[n].id, error: msg }); + } + } + } + + return { embeddedNotes, errors }; + } + + private reportProgress(current: number, total: number, phase: BatchProgress['phase']): void { + if (this.onProgress) { + this.onProgress({ current, total, phase }); + } + } +} diff --git a/src/services/embeddings/ProviderResolver.test.ts b/src/services/embeddings/ProviderResolver.test.ts new file mode 100644 index 0000000..9e81a93 --- /dev/null +++ b/src/services/embeddings/ProviderResolver.test.ts @@ -0,0 +1,47 @@ +import { ProviderResolver } from './ProviderResolver'; +import joplin from 'api'; + +describe('ProviderResolver', () => { + describe('resolve', () => { + it('returns JoplinNativeProvider', () => { + const provider = ProviderResolver.resolve(); + expect(provider).toBeDefined(); + expect(provider.id).toBe('joplin-native'); + expect(provider.modelName).toBe('joplin-native'); + }); + }); + + describe('getDefaultConfig', () => { + it('returns joplin-native as default', () => { + const config = ProviderResolver.getDefaultConfig(); + expect(config.id).toBe('joplin-native'); + }); + }); + + describe('resolveWithValidation', () => { + it('throws when joplin.ai is unavailable', async () => { + (joplin as any).ai = undefined; + await expect(ProviderResolver.resolveWithValidation()).rejects.toThrow( + 'joplin.ai is not available' + ); + }); + + it('throws when index is not ready', async () => { + (joplin as any).ai = { + getIndexStatus: jest.fn().mockResolvedValue({ ready: false, state: 'disabled' }), + }; + await expect(ProviderResolver.resolveWithValidation()).rejects.toThrow( + 'Joplin AI index is not ready' + ); + }); + + it('returns provider when index is ready', async () => { + (joplin as any).ai = { + getIndexStatus: jest.fn().mockResolvedValue({ ready: true, modelId: 'test-model' }), + }; + const provider = await ProviderResolver.resolveWithValidation(); + expect(provider.id).toBe('joplin-native'); + expect(provider.modelName).toBe('test-model'); + }); + }); +}); diff --git a/src/services/embeddings/ProviderResolver.ts b/src/services/embeddings/ProviderResolver.ts new file mode 100644 index 0000000..24ae35d --- /dev/null +++ b/src/services/embeddings/ProviderResolver.ts @@ -0,0 +1,26 @@ +import joplin from 'api'; +import { EmbeddingProvider, ProviderConfig } from './Types'; +import { JoplinNativeProvider } from './Providers/JoplinNativeProvider'; + +export class ProviderResolver { + + public static resolve(): EmbeddingProvider { + return new JoplinNativeProvider(); + } + + public static async resolveWithValidation(): Promise { + const joplinAi = joplin.ai as any; + if (!joplinAi || typeof joplinAi.getIndexStatus !== 'function') { + throw new Error('joplin.ai is not available. Enable AI in Settings → AI. Requires Joplin v3.7+.'); + } + const status = await joplinAi.getIndexStatus(); + if (!status || !status.ready) { + throw new Error('Joplin AI index is not ready. Enable AI and the embedding index in Settings → AI.'); + } + return new JoplinNativeProvider(status.modelId ?? 'joplin-native', 0); + } + + public static getDefaultConfig(): ProviderConfig { + return { id: 'joplin-native' }; + } +} diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.ts b/src/services/embeddings/Providers/JoplinNativeProvider.ts new file mode 100644 index 0000000..90e15b3 --- /dev/null +++ b/src/services/embeddings/Providers/JoplinNativeProvider.ts @@ -0,0 +1,116 @@ +import joplin from 'api'; +import { EmbeddingProvider, ProviderId } from '../Types'; + +export class JoplinNativeProvider implements EmbeddingProvider { + public readonly id: ProviderId = 'joplin-native'; + + private _modelName: string; + private _dimension: number; + private cachedVectors: Map | null = null; + private fetchedModelId: string | null = null; + + public constructor(modelName: string = 'joplin-native', dimension: number = 0) { + this._modelName = modelName; + this._dimension = dimension; + } + + public get modelName(): string { + return this._modelName; + } + + public async fetchVectorsByNoteIds(noteIds: string[]): Promise> { + const joplinAi = joplin.ai as any; + if (!joplinAi || typeof joplinAi.getEmbeddings !== 'function') { + throw new Error('joplin.ai.getEmbeddings is not available. Enable AI in Settings → AI.'); + } + + const status = await joplinAi.getIndexStatus(); + if (!status || !status.ready) { + throw new Error('Joplin AI index is not ready. Wait for indexing to complete or enable AI in Settings → AI.'); + } + + const statusModelId = status.modelId ?? null; + this.fetchedModelId = statusModelId; + this._modelName = statusModelId ?? 'joplin-native'; + + const allChunks: { noteId: string; chunkIndex: number; chunkText: string; vector: number[] }[] = []; + let cursor: string | undefined; + let modelChangeRetries = 0; + const MAX_MODEL_CHANGE_RETRIES = 3; + + do { + const page = await joplinAi.getEmbeddings({ + noteIds: noteIds.length > 0 ? noteIds : undefined, + cursor: cursor, + limit: 1000, + }); + + if (this.fetchedModelId && page.modelId !== this.fetchedModelId) { + modelChangeRetries++; + if (modelChangeRetries > MAX_MODEL_CHANGE_RETRIES) { + throw new Error('Model changed too many times during pagination.'); + } + allChunks.length = 0; + cursor = undefined; + this.fetchedModelId = page.modelId; + this._modelName = page.modelId; + continue; + } + + if (this._dimension === 0 && page.dimension > 0) { + this._dimension = page.dimension; + } + + allChunks.push(...page.chunks); + cursor = page.nextCursor; + } while (cursor); + + const grouped = new Map(); + for (const chunk of allChunks) { + if (!grouped.has(chunk.noteId)) { + grouped.set(chunk.noteId, []); + } + grouped.get(chunk.noteId)!.push(chunk.vector); + } + + const result = new Map(); + for (const [noteId, vectors] of grouped) { + if (vectors.length === 0) continue; + + const dim = vectors[0].length; + const pooled = new Array(dim).fill(0); + for (const vec of vectors) { + for (let i = 0; i < dim; i++) { + pooled[i] += vec[i]; + } + } + for (let i = 0; i < dim; i++) { + pooled[i] /= vectors.length; + } + + let norm = 0; + for (let i = 0; i < dim; i++) { + norm += pooled[i] * pooled[i]; + } + norm = Math.sqrt(norm); + if (norm > 0) { + for (let i = 0; i < dim; i++) { + pooled[i] /= norm; + } + } + + result.set(noteId, pooled); + } + + this.cachedVectors = result; + return result; + } + + public getCachedVectors(): Map | null { + return this.cachedVectors; + } + + public getFetchedModelId(): string | null { + return this.fetchedModelId; + } +} diff --git a/src/services/embeddings/Types.ts b/src/services/embeddings/Types.ts new file mode 100644 index 0000000..bb98018 --- /dev/null +++ b/src/services/embeddings/Types.ts @@ -0,0 +1,31 @@ +import { Note } from '../../data/Types'; + +export type ProviderId = 'joplin-native'; + +export interface EmbeddingProvider { + readonly id: ProviderId; + readonly modelName: string; + fetchVectorsByNoteIds(noteIds: string[]): Promise>; + getCachedVectors?(): Map | null; + getFetchedModelId?(): string | null; +} + +export interface ProviderConfig { + id: ProviderId; +} + +export interface EmbeddedNote { + note: Note; + embedding: number[]; +} + +export interface BatchProgress { + current: number; + total: number; + phase: 'preprocessing' | 'embedding'; +} + +export interface EmbeddingResult { + embeddedNotes: EmbeddedNote[]; + errors: Array<{ noteId: string; error: string }>; +} diff --git a/src/services/graph/GraphBuilder.ts b/src/services/graph/GraphBuilder.ts index 1921cfa..7c8c991 100644 --- a/src/services/graph/GraphBuilder.ts +++ b/src/services/graph/GraphBuilder.ts @@ -9,11 +9,6 @@ export class GraphBuilder { this.edgeFactory = edgeFactory; } - /** - * Builds a graph from enriched notes, creating nodes and edges from links and shared tags. - * @param notes - notes with `links` and `tags` already populated. - * @returns graph data ready for rendering (nodes and edges). - */ public build(notes: Note[]): GraphData { const degreeMap = new Map(); for (const note of notes) { diff --git a/src/tests/mocks/joplin.ts b/src/tests/mocks/joplin.ts index 40ce356..0b6e5d4 100644 --- a/src/tests/mocks/joplin.ts +++ b/src/tests/mocks/joplin.ts @@ -1,9 +1,15 @@ -const data = { - get: jest.fn(), +const joplinAi = { + getIndexStatus: jest.fn(), + getEmbeddings: jest.fn(), + search: jest.fn(), + chat: jest.fn(), }; const joplin = { - data, + data: { + get: jest.fn(), + }, + ai: joplinAi, }; export default joplin; diff --git a/src/ui/graph-view.js b/src/ui/graph-view.js index f5ddfa7..ac6e14c 100644 --- a/src/ui/graph-view.js +++ b/src/ui/graph-view.js @@ -1,7 +1,9 @@ import cytoscape from 'cytoscape'; import fcose from 'cytoscape-fcose'; +import svg from 'cytoscape-svg'; cytoscape.use(fcose); +cytoscape.use(svg); var FCOSE_OPTIONS = { name: 'fcose', @@ -202,12 +204,7 @@ function renderGraph(message) { cy.layout(FCOSE_OPTIONS).run(); - var edgeCount = (message.edges || []).length; - if (edgeCount === 0) { - showStatus(message.nodes.length + ' notes, 0 connections'); - } else { - hideStatus(); - } + hideStatus(); } /** Write counts into the stats bar elements (stat-notes, stat-explicit, stat-semantic, stat-tags). */ @@ -225,7 +222,7 @@ function updateStats(notes, explicit, semantic, tags) { function createExportMenu(btn) { var menu = document.createElement('div'); menu.className = 'export-menu'; - menu.innerHTML = ''; + menu.innerHTML = ''; document.body.appendChild(menu); btn.addEventListener('click', function (e) { @@ -248,6 +245,10 @@ function createExportMenu(btn) { var bg = getComputedStyle(document.body).getPropertyValue('--joplin-background-color').trim() || '#1e1e1e'; if (format === 'png') { downloadFile(cy.png({ full: true, bg: bg }), 'note-graph.png'); + } else if (format === 'svg') { + var svgString = cy.svg({ full: true, bg: bg }); + var svgBlob = new Blob([svgString], { type: 'image/svg+xml;charset=utf-8' }); + downloadFile(URL.createObjectURL(svgBlob), 'note-graph.svg'); } else if (format === 'json') { var blob = new Blob([JSON.stringify(cy.json().elements, null, 2)], { type: 'application/json' }); downloadFile(URL.createObjectURL(blob), 'note-graph.json'); @@ -506,7 +507,10 @@ function init() { if (typeof webviewApi !== 'undefined') { webviewApi.onMessage(function (message) { if (message && message.type === 'graph-data') { - clearInterval(pollTimer); + if (pollTimer) { + clearInterval(pollTimer); + pollTimer = null; + } renderGraph(message); } if (message && message.type === 'fit-to-screen') { diff --git a/src/ui/setup.js b/src/ui/setup.js index 50d9743..fec1b4b 100644 --- a/src/ui/setup.js +++ b/src/ui/setup.js @@ -7,10 +7,14 @@ }); }; + const bindAll = () => { + bindClose(); + }; + if (document.readyState === 'loading') { - document.addEventListener('DOMContentLoaded', bindClose); + document.addEventListener('DOMContentLoaded', bindAll); return; } - bindClose(); + bindAll(); })(); diff --git a/src/ui/styles/panel.css b/src/ui/styles/panel.css index 89cc1c3..0e7b351 100644 --- a/src/ui/styles/panel.css +++ b/src/ui/styles/panel.css @@ -476,4 +476,198 @@ body { .export-menu__item svg { flex-shrink: 0; +} + +/* Settings modal */ + +.settings-overlay { + display: none; + position: fixed; + inset: 0; + background: rgba(0, 0, 0, 0.45); + z-index: 2000; + align-items: center; + justify-content: center; + font-family: -apple-system, BlinkMacSystemFont, sans-serif; +} + +.settings-overlay--visible { + display: flex; +} + +.settings-modal { + background: var(--joplin-background-color); + border-radius: 12px; + box-shadow: 0 8px 40px rgba(0, 0, 0, 0.22); + width: 520px; + max-height: 82vh; + overflow-y: auto; + padding: 0; +} + +.settings-modal__head { + display: flex; + align-items: center; + justify-content: space-between; + padding: 10px 20px; + background: rgba(128, 128, 128, 0.05); + border-bottom: 1px solid rgba(128, 128, 128, 0.12); +} + +.settings-modal__title { + font-size: 15px; + font-weight: 700; + color: var(--joplin-color); + letter-spacing: -0.01em; +} + +.settings-modal__close { + background: transparent; + border: none; + border-radius: 5px; + color: var(--joplin-color-faded, #999); + cursor: pointer; + padding: 4px; + display: flex; + align-items: center; + justify-content: center; + transition: color 0.15s, background 0.15s; +} + +.settings-modal__close:hover { + color: var(--joplin-color); + background: rgba(128, 128, 128, 0.10); +} + +.settings-modal__close svg { + width: 15px; + height: 15px; +} + +.settings-modal__body { + padding: 8px 20px 2px; +} + +.settings-modal__group { + padding: 10px 0 0; +} + +.settings-modal__group:first-child { + padding-top: 0; +} + +.settings-modal__group-head { + font-size: 10px; + font-weight: 600; + color: rgba(128, 128, 128, 0.50); + text-transform: uppercase; + letter-spacing: 0.08em; + margin-bottom: 6px; +} + +.settings-modal__group-divider { + height: 1px; + background: rgba(128, 128, 128, 0.16); + margin-bottom: 10px; +} + +.settings-modal__group-subtitle { + font-size: 9px; + font-weight: 400; + color: rgba(128, 128, 128, 0.50); + text-transform: none; + letter-spacing: 0.02em; + margin-left: 4px; +} + +.settings-modal__row { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: 20px; + padding: 8px 0; +} + +.settings-modal__row + .settings-modal__row { + border-top: 1px solid rgba(128, 128, 128, 0.06); +} + +.settings-modal__row-info { + flex: 1 1 auto; + min-width: 0; +} + +.settings-modal__row-label { + font-size: 11px; + font-weight: 600; + color: var(--joplin-color); +} + +.settings-modal__row-desc { + font-size: 10px; + color: var(--joplin-color-faded, #999); + margin-top: 3px; + line-height: 1.3; +} + +.settings-modal__input, +.settings-modal__select { + background: var(--joplin-background-color3, rgba(128, 128, 128, 0.04)); + border: 1px solid rgba(128, 128, 128, 0.15); + border-radius: 5px; + color: var(--joplin-color); + font-size: 11px; + font-family: inherit; + padding: 5px 8px; + outline: none; + flex-shrink: 0; + width: 195px; + box-sizing: border-box; +} + +.settings-modal__input::placeholder { + color: var(--joplin-color-faded, #888); +} + +.settings-modal__input:focus, +.settings-modal__select:focus { + border-color: rgba(59, 130, 246, 0.35); + background: var(--joplin-background-color3, rgba(128, 128, 128, 0.06)); +} + +.settings-modal__actions { + display: flex; + justify-content: flex-end; + gap: 6px; + padding: 10px 20px 12px; + border-top: 1px solid rgba(128, 128, 128, 0.10); +} + +.settings-modal__btn { + padding: 6px 14px; + border-radius: 5px; + font-size: 11px; + font-weight: 500; + font-family: inherit; + cursor: pointer; + transition: background 0.15s, color 0.15s; + border: none; +} + +.settings-modal__btn--primary { + background: #3b82f6; + color: #fff; +} + +.settings-modal__btn--primary:hover { + background: #2563eb; +} + +.settings-modal__btn--secondary { + background: transparent; + color: var(--joplin-color-faded); +} + +.settings-modal__btn--secondary:hover { + color: var(--joplin-color); } \ No newline at end of file diff --git a/src/ui/webview.ts b/src/ui/webview.ts index 25b72a3..fc735c7 100644 --- a/src/ui/webview.ts +++ b/src/ui/webview.ts @@ -52,9 +52,6 @@ const getPanel = (): ViewHandle => { return panelHandle; }; -/** - * Initializes the note graph panel. Safe to call multiple times (no-op after first). - */ export const initializeAiNoteGraphPanel = async (): Promise => { if (panelHandle) { return; @@ -62,19 +59,11 @@ export const initializeAiNoteGraphPanel = async (): Promise => { panelHandle = await createPanel(); }; -/** - * Shows the note graph panel in the Joplin UI. - */ export const showAiNoteGraphPanel = async (): Promise => { const handle = getPanel(); await joplin.views.panels.show(handle); }; -/** - * Stores graph data and pushes it to the panel if already shown. - * On first call the panel requests the data on load; subsequent calls push proactively. - * @param graphData - the graph nodes and edges to display. - */ export const postGraphData = async (graphData: GraphData): Promise => { const hadData = currentGraphData !== null; currentGraphData = graphData; diff --git a/tsconfig.json b/tsconfig.json index 2120989..a04ae91 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -8,5 +8,6 @@ "baseUrl": ".", "ignoreDeprecations": "6.0", "types": ["jest", "node"] - } + }, + "include": ["src"] } From 7872ce27d20f853db7870a4af8d777e710582707 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Thu, 9 Jul 2026 19:16:39 +0530 Subject: [PATCH 02/28] ANG-007: fixed ai suggestions --- src/services/embeddings/Orchestrator.ts | 1 + src/services/embeddings/Providers/JoplinNativeProvider.ts | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/services/embeddings/Orchestrator.ts b/src/services/embeddings/Orchestrator.ts index 64332e3..33be4f4 100644 --- a/src/services/embeddings/Orchestrator.ts +++ b/src/services/embeddings/Orchestrator.ts @@ -13,6 +13,7 @@ export class EmbeddingOrchestrator { public setProvider(provider: EmbeddingProvider): void { this.provider = provider; + this.cancelled = false; } public setOnProgress(callback: (progress: BatchProgress) => void): void { diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.ts b/src/services/embeddings/Providers/JoplinNativeProvider.ts index 90e15b3..e26e880 100644 --- a/src/services/embeddings/Providers/JoplinNativeProvider.ts +++ b/src/services/embeddings/Providers/JoplinNativeProvider.ts @@ -23,6 +23,9 @@ export class JoplinNativeProvider implements EmbeddingProvider { if (!joplinAi || typeof joplinAi.getEmbeddings !== 'function') { throw new Error('joplin.ai.getEmbeddings is not available. Enable AI in Settings → AI.'); } + if (typeof joplinAi.getIndexStatus !== 'function') { + throw new Error('joplin.ai.getIndexStatus is not available. Enable AI in Settings → AI.'); + } const status = await joplinAi.getIndexStatus(); if (!status || !status.ready) { @@ -52,8 +55,8 @@ export class JoplinNativeProvider implements EmbeddingProvider { } allChunks.length = 0; cursor = undefined; - this.fetchedModelId = page.modelId; - this._modelName = page.modelId; + this.fetchedModelId = page.modelId ?? null; + this._modelName = page.modelId ?? this._modelName; continue; } From 3df50259e4a76b6fb4c4c6dedebb1bc5b631e96d Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Thu, 9 Jul 2026 19:44:52 +0530 Subject: [PATCH 03/28] ANG-007: bug fix, added doc strings & few more tests --- src/services/embeddings/Orchestrator.test.ts | 29 +++++ src/services/embeddings/Orchestrator.ts | 4 + .../embeddings/ProviderResolver.test.ts | 8 ++ src/services/embeddings/ProviderResolver.ts | 4 + .../Providers/JoplinNativeProvider.test.ts | 120 ++++++++++++++++++ .../Providers/JoplinNativeProvider.ts | 84 ++++++++---- src/services/similarity/EdgeFactory.test.ts | 23 ++++ 7 files changed, 248 insertions(+), 24 deletions(-) create mode 100644 src/services/embeddings/Providers/JoplinNativeProvider.test.ts diff --git a/src/services/embeddings/Orchestrator.test.ts b/src/services/embeddings/Orchestrator.test.ts index 5e0587e..a8652b0 100644 --- a/src/services/embeddings/Orchestrator.test.ts +++ b/src/services/embeddings/Orchestrator.test.ts @@ -1,4 +1,5 @@ import { EmbeddingOrchestrator } from './Orchestrator'; +import { BatchProgress } from './Types'; import { Note } from '../../data/Types'; function makeNote(id: string, title: string, body: string): Note { @@ -72,6 +73,34 @@ describe('EmbeddingOrchestrator', () => { expect(result.errors[0].noteId).toBe('n2'); }); + it('emits progress updates while embedding', async () => { + const progressUpdates: BatchProgress[] = []; + orchestrator.setOnProgress((progress) => { + progressUpdates.push(progress); + }); + + const mockVectors = new Map(); + mockVectors.set('n1', [0.1, 0.2, 0.3]); + mockVectors.set('n2', [0.4, 0.5, 0.6]); + + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'test-model', + fetchVectorsByNoteIds: jest.fn().mockResolvedValue(mockVectors), + }); + + await orchestrator.embedNotes([ + makeNote('n1', 'T1', 'B1'), + makeNote('n2', 'T2', 'B2'), + ]); + + expect(progressUpdates).toEqual([ + { current: 0, total: 2, phase: 'embedding' }, + { current: 1, total: 2, phase: 'embedding' }, + { current: 2, total: 2, phase: 'embedding' }, + ]); + }); + it('returns empty when cancelled', async () => { orchestrator.setProvider({ id: 'joplin-native', diff --git a/src/services/embeddings/Orchestrator.ts b/src/services/embeddings/Orchestrator.ts index 33be4f4..64e5661 100644 --- a/src/services/embeddings/Orchestrator.ts +++ b/src/services/embeddings/Orchestrator.ts @@ -24,6 +24,10 @@ export class EmbeddingOrchestrator { this.cancelled = true; } + /** + * Embeds the provided notes, preserving note order and reporting missing + * embeddings as per-note errors. + */ public async embedNotes(notes: Note[]): Promise { const embeddedNotes: EmbeddedNote[] = []; const errors: Array<{ noteId: string; error: string }> = []; diff --git a/src/services/embeddings/ProviderResolver.test.ts b/src/services/embeddings/ProviderResolver.test.ts index 9e81a93..242af28 100644 --- a/src/services/embeddings/ProviderResolver.test.ts +++ b/src/services/embeddings/ProviderResolver.test.ts @@ -43,5 +43,13 @@ describe('ProviderResolver', () => { expect(provider.id).toBe('joplin-native'); expect(provider.modelName).toBe('test-model'); }); + + it('uses the default native model when index status omits modelId', async () => { + (joplin as any).ai = { + getIndexStatus: jest.fn().mockResolvedValue({ ready: true }), + }; + const provider = await ProviderResolver.resolveWithValidation(); + expect(provider.modelName).toBe('joplin-native'); + }); }); }); diff --git a/src/services/embeddings/ProviderResolver.ts b/src/services/embeddings/ProviderResolver.ts index 24ae35d..76e39a6 100644 --- a/src/services/embeddings/ProviderResolver.ts +++ b/src/services/embeddings/ProviderResolver.ts @@ -8,6 +8,10 @@ export class ProviderResolver { return new JoplinNativeProvider(); } + /** + * Resolves the native embedding provider after verifying that Joplin AI is + * available and its embedding index is ready. + */ public static async resolveWithValidation(): Promise { const joplinAi = joplin.ai as any; if (!joplinAi || typeof joplinAi.getIndexStatus !== 'function') { diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.test.ts b/src/services/embeddings/Providers/JoplinNativeProvider.test.ts new file mode 100644 index 0000000..63130a2 --- /dev/null +++ b/src/services/embeddings/Providers/JoplinNativeProvider.test.ts @@ -0,0 +1,120 @@ +import joplin from 'api'; +import { JoplinNativeProvider } from './JoplinNativeProvider'; + +describe('JoplinNativeProvider', () => { + it('returns an empty map without touching AI when no note ids are requested', async () => { + const provider = new JoplinNativeProvider(); + const ai = joplin.ai as unknown as { + getIndexStatus: jest.Mock; + getEmbeddings: jest.Mock; + }; + + ai.getIndexStatus.mockResolvedValue({ ready: true, modelId: 'test-model' }); + ai.getEmbeddings.mockResolvedValue({ + modelId: 'test-model', + dimension: 3, + chunks: [], + }); + + const vectors = await provider.fetchVectorsByNoteIds([]); + + expect(vectors.size).toBe(0); + expect(ai.getIndexStatus).not.toHaveBeenCalled(); + expect(ai.getEmbeddings).not.toHaveBeenCalled(); + }); + + it('clears stale cached state before starting a new fetch', async () => { + const provider = new JoplinNativeProvider(); + const ai = joplin.ai as unknown as { + getIndexStatus: jest.Mock; + getEmbeddings: jest.Mock; + }; + + ai.getIndexStatus.mockResolvedValue({ ready: true, modelId: 'fresh-model' }); + ai.getEmbeddings.mockResolvedValue({ + modelId: 'fresh-model', + dimension: 2, + chunks: [{ noteId: 'n1', vector: [1, 0] }], + nextCursor: undefined, + }); + + await provider.fetchVectorsByNoteIds(['n1']); + expect(provider.getFetchedModelId()).toBe('fresh-model'); + expect(provider.getCachedVectors()).toEqual(new Map([['n1', [1, 0]]])); + + ai.getEmbeddings.mockRejectedValue(new Error('network error')); + + await expect(provider.fetchVectorsByNoteIds(['n2'])).rejects.toThrow('network error'); + expect(provider.getFetchedModelId()).toBeNull(); + expect(provider.getCachedVectors()).toBeNull(); + }); + + it('pools vectors across pages and normalizes the result', async () => { + const provider = new JoplinNativeProvider(); + const ai = joplin.ai as unknown as { + getIndexStatus: jest.Mock; + getEmbeddings: jest.Mock; + }; + + ai.getIndexStatus.mockResolvedValue({ ready: true, modelId: 'test-model' }); + ai.getEmbeddings + .mockResolvedValueOnce({ + modelId: 'test-model', + dimension: 2, + chunks: [{ noteId: 'n1', vector: [3, 0] }], + nextCursor: 'cursor-1', + }) + .mockResolvedValueOnce({ + modelId: 'test-model', + dimension: 2, + chunks: [{ noteId: 'n1', vector: [0, 4] }], + nextCursor: undefined, + }); + + const vectors = await provider.fetchVectorsByNoteIds(['n1']); + const vector = vectors.get('n1'); + + expect(ai.getEmbeddings).toHaveBeenCalledTimes(2); + expect(vector).toBeDefined(); + expect(vector![0]).toBeCloseTo(0.6, 5); + expect(vector![1]).toBeCloseTo(0.8, 5); + expect(provider.getFetchedModelId()).toBe('test-model'); + }); + + it('restarts pagination when the model changes mid-fetch', async () => { + const provider = new JoplinNativeProvider(); + const ai = joplin.ai as unknown as { + getIndexStatus: jest.Mock; + getEmbeddings: jest.Mock; + }; + + ai.getIndexStatus.mockResolvedValue({ ready: true, modelId: 'model-a' }); + ai.getEmbeddings + .mockResolvedValueOnce({ + modelId: 'model-a', + dimension: 2, + chunks: [{ noteId: 'n1', vector: [1, 0] }], + nextCursor: 'cursor-1', + }) + .mockResolvedValueOnce({ + modelId: 'model-b', + dimension: 2, + chunks: [{ noteId: 'n1', vector: [0, 1] }], + nextCursor: undefined, + }) + .mockResolvedValueOnce({ + modelId: 'model-b', + dimension: 2, + chunks: [{ noteId: 'n1', vector: [0, 1] }], + nextCursor: undefined, + }); + + const vectors = await provider.fetchVectorsByNoteIds(['n1']); + const vector = vectors.get('n1'); + + expect(ai.getEmbeddings).toHaveBeenCalledTimes(3); + expect(vector).toEqual([0, 1]); + expect(provider.getFetchedModelId()).toBe('model-b'); + expect(provider.modelName).toBe('model-b'); + }); +}); diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.ts b/src/services/embeddings/Providers/JoplinNativeProvider.ts index e26e880..b928c14 100644 --- a/src/services/embeddings/Providers/JoplinNativeProvider.ts +++ b/src/services/embeddings/Providers/JoplinNativeProvider.ts @@ -1,6 +1,20 @@ import joplin from 'api'; import { EmbeddingProvider, ProviderId } from '../Types'; +interface JoplinAiApi { + getIndexStatus: () => Promise<{ ready: boolean; modelId?: string | null }>; + getEmbeddings: (params: { + noteIds?: string[]; + cursor?: string; + limit: number; + }) => Promise<{ + modelId?: string | null; + dimension: number; + chunks: Array<{ noteId: string; vector: number[] }>; + nextCursor?: string; + }>; +} + export class JoplinNativeProvider implements EmbeddingProvider { public readonly id: ProviderId = 'joplin-native'; @@ -18,8 +32,19 @@ export class JoplinNativeProvider implements EmbeddingProvider { return this._modelName; } + /** + * Fetches embeddings for the requested note IDs and pools repeated chunks + * into one normalized vector per note. + */ public async fetchVectorsByNoteIds(noteIds: string[]): Promise> { - const joplinAi = joplin.ai as any; + if (noteIds.length === 0) { + return new Map(); + } + + this.cachedVectors = null; + this.fetchedModelId = null; + + const joplinAi = joplin.ai as unknown as JoplinAiApi | undefined; if (!joplinAi || typeof joplinAi.getEmbeddings !== 'function') { throw new Error('joplin.ai.getEmbeddings is not available. Enable AI in Settings → AI.'); } @@ -32,50 +57,61 @@ export class JoplinNativeProvider implements EmbeddingProvider { throw new Error('Joplin AI index is not ready. Wait for indexing to complete or enable AI in Settings → AI.'); } - const statusModelId = status.modelId ?? null; - this.fetchedModelId = statusModelId; - this._modelName = statusModelId ?? 'joplin-native'; + let trackedModelId: string | null = status.modelId ?? null; + this._modelName = trackedModelId ?? 'joplin-native'; - const allChunks: { noteId: string; chunkIndex: number; chunkText: string; vector: number[] }[] = []; + const grouped = new Map(); let cursor: string | undefined; let modelChangeRetries = 0; const MAX_MODEL_CHANGE_RETRIES = 3; - do { + while (true) { const page = await joplinAi.getEmbeddings({ - noteIds: noteIds.length > 0 ? noteIds : undefined, + noteIds: noteIds, cursor: cursor, limit: 1000, }); - if (this.fetchedModelId && page.modelId !== this.fetchedModelId) { - modelChangeRetries++; - if (modelChangeRetries > MAX_MODEL_CHANGE_RETRIES) { - throw new Error('Model changed too many times during pagination.'); + const pageModelId = page.modelId ?? null; + + if (pageModelId) { + if (!trackedModelId) { + trackedModelId = pageModelId; + this._modelName = pageModelId; + } else if (pageModelId !== trackedModelId) { + modelChangeRetries++; + if (modelChangeRetries > MAX_MODEL_CHANGE_RETRIES) { + throw new Error('Model changed too many times during pagination.'); + } + trackedModelId = pageModelId; + this._modelName = pageModelId; + grouped.clear(); + cursor = undefined; + continue; } - allChunks.length = 0; - cursor = undefined; - this.fetchedModelId = page.modelId ?? null; - this._modelName = page.modelId ?? this._modelName; - continue; } if (this._dimension === 0 && page.dimension > 0) { this._dimension = page.dimension; } - allChunks.push(...page.chunks); - cursor = page.nextCursor; - } while (cursor); + for (const chunk of page.chunks) { + const list = grouped.get(chunk.noteId); + if (list) { + list.push(chunk.vector); + } else { + grouped.set(chunk.noteId, [chunk.vector]); + } + } - const grouped = new Map(); - for (const chunk of allChunks) { - if (!grouped.has(chunk.noteId)) { - grouped.set(chunk.noteId, []); + cursor = page.nextCursor; + if (!cursor) { + break; } - grouped.get(chunk.noteId)!.push(chunk.vector); } + this.fetchedModelId = trackedModelId; + const result = new Map(); for (const [noteId, vectors] of grouped) { if (vectors.length === 0) continue; diff --git a/src/services/similarity/EdgeFactory.test.ts b/src/services/similarity/EdgeFactory.test.ts index 90c1924..e8498df 100644 --- a/src/services/similarity/EdgeFactory.test.ts +++ b/src/services/similarity/EdgeFactory.test.ts @@ -34,6 +34,21 @@ describe('EdgeFactory', () => { expect(factory.createEdges([note('a', 'A'), note('b', 'B')])).toEqual([]); }); + it('handles missing links and tags arrays', () => { + const edges = factory.createEdges([ + { + id: 'a', + parent_id: 'p1', + title: 'A', + body: '', + created_time: 0, + updated_time: 1, + } as Note, + ]); + + expect(edges).toEqual([]); + }); + it('ignores resource links that are not note IDs', () => { const notes = [ note('a', 'A', ['resource123']), @@ -101,4 +116,12 @@ describe('EdgeFactory', () => { it('ignores self-referencing links', () => { expect(factory.createEdges([note('a', 'A', ['a'])])).toEqual([]); }); + + it('skips tags that connect too many notes', () => { + const notes = Array.from({ length: 21 }, (_, i) => + note(`n${i}`, `N${i}`, [], ['common']) + ); + + expect(factory.createEdges(notes)).toEqual([]); + }); }); From 731889fe075adfa6fa301c6b3799afe15d870955 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Thu, 9 Jul 2026 22:05:02 +0530 Subject: [PATCH 04/28] ANG-007:add docstrings --- src/index.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index d6a99c0..98024b8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -17,7 +17,10 @@ const registerSettings = async (): Promise => { description: 'Uses Joplin\'s built-in AI to discover connections between your notes. Enable AI in Settings → AI.', }); }; - +/** + * Loads all notes from the Joplin API and enriches them with links and tags. + * @returns enriched notes ready for graph building. + */ export const loadNotes = async (): Promise => { const noteRepository = new NoteRepository(); const { notes } = await noteRepository.getAllNotes(); From e43e39095123cd1989ce40aa9b924d556c1b9f00 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Thu, 9 Jul 2026 22:14:08 +0530 Subject: [PATCH 05/28] ANG-007:remove settings section --- src/index.ts | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/index.ts b/src/index.ts index 98024b8..af8d9da 100644 --- a/src/index.ts +++ b/src/index.ts @@ -8,15 +8,7 @@ import { GraphBuilder } from './services/graph/GraphBuilder'; const SHOW_NOTE_GRAPH_COMMAND = 'showNoteGraph'; const SHOW_NOTE_GRAPH_MENU_ITEM = 'showNoteGraphMenuItem'; -const SETTINGS_SECTION = 'noteGraphSection'; -const registerSettings = async (): Promise => { - await joplin.settings.registerSection(SETTINGS_SECTION, { - label: 'Note Graph', - iconName: 'fas fa-project-diagram', - description: 'Uses Joplin\'s built-in AI to discover connections between your notes. Enable AI in Settings → AI.', - }); -}; /** * Loads all notes from the Joplin API and enriches them with links and tags. * @returns enriched notes ready for graph building. @@ -62,7 +54,6 @@ const registerMenuItems = async (): Promise => { joplin.plugins.register({ onStart: async function () { console.info('Note Graph plugin started.'); - await registerSettings(); await initializeAiNoteGraphPanel(); await registerCommands(); await registerMenuItems(); From c7d92ff4775ef95dba8733d65f2dbb6b65fc211e Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Sun, 12 Jul 2026 14:01:55 +0530 Subject: [PATCH 06/28] ANG-007:split into three helpers --- .../Providers/JoplinNativeProvider.ts | 61 +++++++++++++------ 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.ts b/src/services/embeddings/Providers/JoplinNativeProvider.ts index b928c14..ce472ad 100644 --- a/src/services/embeddings/Providers/JoplinNativeProvider.ts +++ b/src/services/embeddings/Providers/JoplinNativeProvider.ts @@ -32,10 +32,6 @@ export class JoplinNativeProvider implements EmbeddingProvider { return this._modelName; } - /** - * Fetches embeddings for the requested note IDs and pools repeated chunks - * into one normalized vector per note. - */ public async fetchVectorsByNoteIds(noteIds: string[]): Promise> { if (noteIds.length === 0) { return new Map(); @@ -44,15 +40,47 @@ export class JoplinNativeProvider implements EmbeddingProvider { this.cachedVectors = null; this.fetchedModelId = null; - const joplinAi = joplin.ai as unknown as JoplinAiApi | undefined; - if (!joplinAi || typeof joplinAi.getEmbeddings !== 'function') { + const api = this.validateAiApi(); + const grouped = await this.fetchAllPages(api, noteIds); + + this.fetchedModelId = this._modelName; + + const result = this.poolAndNormalize(grouped); + this.cachedVectors = result; + return result; + } + + public getCachedVectors(): Map | null { + return this.cachedVectors; + } + + public getFetchedModelId(): string | null { + return this.fetchedModelId; + } + + /** Checks that joplin.ai exists and has the required methods. */ + private validateAiApi(): JoplinAiApi { + const api = joplin.ai as unknown as JoplinAiApi | undefined; + + if (!api || typeof api.getEmbeddings !== 'function') { throw new Error('joplin.ai.getEmbeddings is not available. Enable AI in Settings → AI.'); } - if (typeof joplinAi.getIndexStatus !== 'function') { + if (typeof api.getIndexStatus !== 'function') { throw new Error('joplin.ai.getIndexStatus is not available. Enable AI in Settings → AI.'); } - const status = await joplinAi.getIndexStatus(); + return api; + } + + /** + * Pages through getEmbeddings collecting vectors per note. + * Restarts pagination if the embedding model changes mid-fetch. + */ + private async fetchAllPages( + api: JoplinAiApi, + noteIds: string[], + ): Promise> { + const status = await api.getIndexStatus(); if (!status || !status.ready) { throw new Error('Joplin AI index is not ready. Wait for indexing to complete or enable AI in Settings → AI.'); } @@ -66,7 +94,7 @@ export class JoplinNativeProvider implements EmbeddingProvider { const MAX_MODEL_CHANGE_RETRIES = 3; while (true) { - const page = await joplinAi.getEmbeddings({ + const page = await api.getEmbeddings({ noteIds: noteIds, cursor: cursor, limit: 1000, @@ -110,9 +138,13 @@ export class JoplinNativeProvider implements EmbeddingProvider { } } - this.fetchedModelId = trackedModelId; + return grouped; + } + /** Averages multiple chunk vectors per note into one vector and L2-normalizes. */ + private poolAndNormalize(grouped: Map): Map { const result = new Map(); + for (const [noteId, vectors] of grouped) { if (vectors.length === 0) continue; @@ -141,15 +173,6 @@ export class JoplinNativeProvider implements EmbeddingProvider { result.set(noteId, pooled); } - this.cachedVectors = result; return result; } - - public getCachedVectors(): Map | null { - return this.cachedVectors; - } - - public getFetchedModelId(): string | null { - return this.fetchedModelId; - } } From c901b8454543718bb0a557ecb9ab080a6ac49719 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Sun, 12 Jul 2026 14:06:33 +0530 Subject: [PATCH 07/28] ANG-007:this._modelName set once after loop ends --- src/services/embeddings/Providers/JoplinNativeProvider.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.ts b/src/services/embeddings/Providers/JoplinNativeProvider.ts index ce472ad..9177a8b 100644 --- a/src/services/embeddings/Providers/JoplinNativeProvider.ts +++ b/src/services/embeddings/Providers/JoplinNativeProvider.ts @@ -86,7 +86,6 @@ export class JoplinNativeProvider implements EmbeddingProvider { } let trackedModelId: string | null = status.modelId ?? null; - this._modelName = trackedModelId ?? 'joplin-native'; const grouped = new Map(); let cursor: string | undefined; @@ -105,14 +104,12 @@ export class JoplinNativeProvider implements EmbeddingProvider { if (pageModelId) { if (!trackedModelId) { trackedModelId = pageModelId; - this._modelName = pageModelId; } else if (pageModelId !== trackedModelId) { modelChangeRetries++; if (modelChangeRetries > MAX_MODEL_CHANGE_RETRIES) { throw new Error('Model changed too many times during pagination.'); } trackedModelId = pageModelId; - this._modelName = pageModelId; grouped.clear(); cursor = undefined; continue; @@ -138,6 +135,7 @@ export class JoplinNativeProvider implements EmbeddingProvider { } } + this._modelName = trackedModelId ?? 'joplin-native'; return grouped; } From 7b301e11281c5b253bf00e9455b900df7edcc685 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Sun, 12 Jul 2026 14:09:06 +0530 Subject: [PATCH 08/28] ANG-007:added docstring --- src/services/graph/GraphBuilder.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/services/graph/GraphBuilder.ts b/src/services/graph/GraphBuilder.ts index 7c8c991..6d1beb0 100644 --- a/src/services/graph/GraphBuilder.ts +++ b/src/services/graph/GraphBuilder.ts @@ -8,7 +8,11 @@ export class GraphBuilder { public constructor(edgeFactory = new EdgeFactory()) { this.edgeFactory = edgeFactory; } - + /** + * Builds a graph from enriched notes, creating nodes and edges from links and shared tags. + * @param notes - notes with `links` and `tags` already populated. + * @returns graph data ready for rendering (nodes and edges). + */ public build(notes: Note[]): GraphData { const degreeMap = new Map(); for (const note of notes) { From 1b806c87bcb08acd4bf4af4dd7fa67ec0880a0f0 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Sun, 12 Jul 2026 14:09:48 +0530 Subject: [PATCH 09/28] ANG-007:added docstring --- src/services/graph/GraphBuilder.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/services/graph/GraphBuilder.ts b/src/services/graph/GraphBuilder.ts index 6d1beb0..1921cfa 100644 --- a/src/services/graph/GraphBuilder.ts +++ b/src/services/graph/GraphBuilder.ts @@ -8,6 +8,7 @@ export class GraphBuilder { public constructor(edgeFactory = new EdgeFactory()) { this.edgeFactory = edgeFactory; } + /** * Builds a graph from enriched notes, creating nodes and edges from links and shared tags. * @param notes - notes with `links` and `tags` already populated. From db085bff9ecfbdadf7e4d2d8ac7964bbc886ee45 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Sun, 12 Jul 2026 14:31:48 +0530 Subject: [PATCH 10/28] ANG-007: JoplinAiApi is exported and reused in ProviderResolver --- src/services/embeddings/ProviderResolver.ts | 4 ++-- src/services/embeddings/Providers/JoplinNativeProvider.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/embeddings/ProviderResolver.ts b/src/services/embeddings/ProviderResolver.ts index 76e39a6..6e9eb32 100644 --- a/src/services/embeddings/ProviderResolver.ts +++ b/src/services/embeddings/ProviderResolver.ts @@ -1,6 +1,6 @@ import joplin from 'api'; import { EmbeddingProvider, ProviderConfig } from './Types'; -import { JoplinNativeProvider } from './Providers/JoplinNativeProvider'; +import { JoplinNativeProvider, JoplinAiApi } from './Providers/JoplinNativeProvider'; export class ProviderResolver { @@ -13,7 +13,7 @@ export class ProviderResolver { * available and its embedding index is ready. */ public static async resolveWithValidation(): Promise { - const joplinAi = joplin.ai as any; + const joplinAi = joplin.ai as unknown as JoplinAiApi | undefined; if (!joplinAi || typeof joplinAi.getIndexStatus !== 'function') { throw new Error('joplin.ai is not available. Enable AI in Settings → AI. Requires Joplin v3.7+.'); } diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.ts b/src/services/embeddings/Providers/JoplinNativeProvider.ts index 9177a8b..3c10d62 100644 --- a/src/services/embeddings/Providers/JoplinNativeProvider.ts +++ b/src/services/embeddings/Providers/JoplinNativeProvider.ts @@ -1,7 +1,7 @@ import joplin from 'api'; import { EmbeddingProvider, ProviderId } from '../Types'; -interface JoplinAiApi { +export interface JoplinAiApi { getIndexStatus: () => Promise<{ ready: boolean; modelId?: string | null }>; getEmbeddings: (params: { noteIds?: string[]; From 7648493b9122179045866cae4ef2897b346ab511 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Sun, 12 Jul 2026 15:06:25 +0530 Subject: [PATCH 11/28] ANG-007: moved joplin-native and 1000 to named class constants --- src/services/embeddings/ProviderResolver.ts | 2 +- src/services/embeddings/Providers/JoplinNativeProvider.ts | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/services/embeddings/ProviderResolver.ts b/src/services/embeddings/ProviderResolver.ts index 6e9eb32..0c864a3 100644 --- a/src/services/embeddings/ProviderResolver.ts +++ b/src/services/embeddings/ProviderResolver.ts @@ -21,7 +21,7 @@ export class ProviderResolver { if (!status || !status.ready) { throw new Error('Joplin AI index is not ready. Enable AI and the embedding index in Settings → AI.'); } - return new JoplinNativeProvider(status.modelId ?? 'joplin-native', 0); + return new JoplinNativeProvider(status.modelId ?? JoplinNativeProvider.DEFAULT_MODEL_ID, 0); } public static getDefaultConfig(): ProviderConfig { diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.ts b/src/services/embeddings/Providers/JoplinNativeProvider.ts index 3c10d62..505018b 100644 --- a/src/services/embeddings/Providers/JoplinNativeProvider.ts +++ b/src/services/embeddings/Providers/JoplinNativeProvider.ts @@ -17,13 +17,15 @@ export interface JoplinAiApi { export class JoplinNativeProvider implements EmbeddingProvider { public readonly id: ProviderId = 'joplin-native'; + public static readonly DEFAULT_MODEL_ID = 'joplin-native'; + private static readonly PAGE_SIZE = 1000; private _modelName: string; private _dimension: number; private cachedVectors: Map | null = null; private fetchedModelId: string | null = null; - public constructor(modelName: string = 'joplin-native', dimension: number = 0) { + public constructor(modelName: string = JoplinNativeProvider.DEFAULT_MODEL_ID, dimension: number = 0) { this._modelName = modelName; this._dimension = dimension; } @@ -96,7 +98,7 @@ export class JoplinNativeProvider implements EmbeddingProvider { const page = await api.getEmbeddings({ noteIds: noteIds, cursor: cursor, - limit: 1000, + limit: JoplinNativeProvider.PAGE_SIZE, }); const pageModelId = page.modelId ?? null; @@ -135,7 +137,7 @@ export class JoplinNativeProvider implements EmbeddingProvider { } } - this._modelName = trackedModelId ?? 'joplin-native'; + this._modelName = trackedModelId ?? JoplinNativeProvider.DEFAULT_MODEL_ID; return grouped; } From 520f3ca8cfc6e6256428fd3661cecdeae7fdc69b Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Sun, 12 Jul 2026 15:14:41 +0530 Subject: [PATCH 12/28] ANG-007: revert few unrelated changes --- src/ui/setup.js | 8 +- src/ui/styles/panel.css | 194 ---------------------------------------- src/ui/webview.ts | 11 +++ tsconfig.json | 3 +- 4 files changed, 14 insertions(+), 202 deletions(-) diff --git a/src/ui/setup.js b/src/ui/setup.js index fec1b4b..50d9743 100644 --- a/src/ui/setup.js +++ b/src/ui/setup.js @@ -7,14 +7,10 @@ }); }; - const bindAll = () => { - bindClose(); - }; - if (document.readyState === 'loading') { - document.addEventListener('DOMContentLoaded', bindAll); + document.addEventListener('DOMContentLoaded', bindClose); return; } - bindAll(); + bindClose(); })(); diff --git a/src/ui/styles/panel.css b/src/ui/styles/panel.css index 0e7b351..89cc1c3 100644 --- a/src/ui/styles/panel.css +++ b/src/ui/styles/panel.css @@ -476,198 +476,4 @@ body { .export-menu__item svg { flex-shrink: 0; -} - -/* Settings modal */ - -.settings-overlay { - display: none; - position: fixed; - inset: 0; - background: rgba(0, 0, 0, 0.45); - z-index: 2000; - align-items: center; - justify-content: center; - font-family: -apple-system, BlinkMacSystemFont, sans-serif; -} - -.settings-overlay--visible { - display: flex; -} - -.settings-modal { - background: var(--joplin-background-color); - border-radius: 12px; - box-shadow: 0 8px 40px rgba(0, 0, 0, 0.22); - width: 520px; - max-height: 82vh; - overflow-y: auto; - padding: 0; -} - -.settings-modal__head { - display: flex; - align-items: center; - justify-content: space-between; - padding: 10px 20px; - background: rgba(128, 128, 128, 0.05); - border-bottom: 1px solid rgba(128, 128, 128, 0.12); -} - -.settings-modal__title { - font-size: 15px; - font-weight: 700; - color: var(--joplin-color); - letter-spacing: -0.01em; -} - -.settings-modal__close { - background: transparent; - border: none; - border-radius: 5px; - color: var(--joplin-color-faded, #999); - cursor: pointer; - padding: 4px; - display: flex; - align-items: center; - justify-content: center; - transition: color 0.15s, background 0.15s; -} - -.settings-modal__close:hover { - color: var(--joplin-color); - background: rgba(128, 128, 128, 0.10); -} - -.settings-modal__close svg { - width: 15px; - height: 15px; -} - -.settings-modal__body { - padding: 8px 20px 2px; -} - -.settings-modal__group { - padding: 10px 0 0; -} - -.settings-modal__group:first-child { - padding-top: 0; -} - -.settings-modal__group-head { - font-size: 10px; - font-weight: 600; - color: rgba(128, 128, 128, 0.50); - text-transform: uppercase; - letter-spacing: 0.08em; - margin-bottom: 6px; -} - -.settings-modal__group-divider { - height: 1px; - background: rgba(128, 128, 128, 0.16); - margin-bottom: 10px; -} - -.settings-modal__group-subtitle { - font-size: 9px; - font-weight: 400; - color: rgba(128, 128, 128, 0.50); - text-transform: none; - letter-spacing: 0.02em; - margin-left: 4px; -} - -.settings-modal__row { - display: flex; - align-items: flex-start; - justify-content: space-between; - gap: 20px; - padding: 8px 0; -} - -.settings-modal__row + .settings-modal__row { - border-top: 1px solid rgba(128, 128, 128, 0.06); -} - -.settings-modal__row-info { - flex: 1 1 auto; - min-width: 0; -} - -.settings-modal__row-label { - font-size: 11px; - font-weight: 600; - color: var(--joplin-color); -} - -.settings-modal__row-desc { - font-size: 10px; - color: var(--joplin-color-faded, #999); - margin-top: 3px; - line-height: 1.3; -} - -.settings-modal__input, -.settings-modal__select { - background: var(--joplin-background-color3, rgba(128, 128, 128, 0.04)); - border: 1px solid rgba(128, 128, 128, 0.15); - border-radius: 5px; - color: var(--joplin-color); - font-size: 11px; - font-family: inherit; - padding: 5px 8px; - outline: none; - flex-shrink: 0; - width: 195px; - box-sizing: border-box; -} - -.settings-modal__input::placeholder { - color: var(--joplin-color-faded, #888); -} - -.settings-modal__input:focus, -.settings-modal__select:focus { - border-color: rgba(59, 130, 246, 0.35); - background: var(--joplin-background-color3, rgba(128, 128, 128, 0.06)); -} - -.settings-modal__actions { - display: flex; - justify-content: flex-end; - gap: 6px; - padding: 10px 20px 12px; - border-top: 1px solid rgba(128, 128, 128, 0.10); -} - -.settings-modal__btn { - padding: 6px 14px; - border-radius: 5px; - font-size: 11px; - font-weight: 500; - font-family: inherit; - cursor: pointer; - transition: background 0.15s, color 0.15s; - border: none; -} - -.settings-modal__btn--primary { - background: #3b82f6; - color: #fff; -} - -.settings-modal__btn--primary:hover { - background: #2563eb; -} - -.settings-modal__btn--secondary { - background: transparent; - color: var(--joplin-color-faded); -} - -.settings-modal__btn--secondary:hover { - color: var(--joplin-color); } \ No newline at end of file diff --git a/src/ui/webview.ts b/src/ui/webview.ts index fc735c7..25b72a3 100644 --- a/src/ui/webview.ts +++ b/src/ui/webview.ts @@ -52,6 +52,9 @@ const getPanel = (): ViewHandle => { return panelHandle; }; +/** + * Initializes the note graph panel. Safe to call multiple times (no-op after first). + */ export const initializeAiNoteGraphPanel = async (): Promise => { if (panelHandle) { return; @@ -59,11 +62,19 @@ export const initializeAiNoteGraphPanel = async (): Promise => { panelHandle = await createPanel(); }; +/** + * Shows the note graph panel in the Joplin UI. + */ export const showAiNoteGraphPanel = async (): Promise => { const handle = getPanel(); await joplin.views.panels.show(handle); }; +/** + * Stores graph data and pushes it to the panel if already shown. + * On first call the panel requests the data on load; subsequent calls push proactively. + * @param graphData - the graph nodes and edges to display. + */ export const postGraphData = async (graphData: GraphData): Promise => { const hadData = currentGraphData !== null; currentGraphData = graphData; diff --git a/tsconfig.json b/tsconfig.json index a04ae91..2120989 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -8,6 +8,5 @@ "baseUrl": ".", "ignoreDeprecations": "6.0", "types": ["jest", "node"] - }, - "include": ["src"] + } } From 45a7e2555f30d788aae741431c93331d6e87a7d0 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Sun, 12 Jul 2026 15:23:21 +0530 Subject: [PATCH 13/28] ANG-007: removed unrelated changes --- package.json | 4 ++-- src/services/similarity/EdgeFactory.test.ts | 23 --------------------- 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/package.json b/package.json index a9893d3..47e88e3 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ ], "devDependencies": { "@types/jest": "^29.5.14", - "@types/node": "^18.7.13", + "@types/node": "^18.19.130", "chalk": "^4.1.0", "copy-webpack-plugin": "^11.0.0", "fs-extra": "^10.1.0", @@ -41,4 +41,4 @@ "cytoscape-fcose": "^2.2.0", "cytoscape-svg": "^0.4.0" } -} \ No newline at end of file +} diff --git a/src/services/similarity/EdgeFactory.test.ts b/src/services/similarity/EdgeFactory.test.ts index e8498df..90c1924 100644 --- a/src/services/similarity/EdgeFactory.test.ts +++ b/src/services/similarity/EdgeFactory.test.ts @@ -34,21 +34,6 @@ describe('EdgeFactory', () => { expect(factory.createEdges([note('a', 'A'), note('b', 'B')])).toEqual([]); }); - it('handles missing links and tags arrays', () => { - const edges = factory.createEdges([ - { - id: 'a', - parent_id: 'p1', - title: 'A', - body: '', - created_time: 0, - updated_time: 1, - } as Note, - ]); - - expect(edges).toEqual([]); - }); - it('ignores resource links that are not note IDs', () => { const notes = [ note('a', 'A', ['resource123']), @@ -116,12 +101,4 @@ describe('EdgeFactory', () => { it('ignores self-referencing links', () => { expect(factory.createEdges([note('a', 'A', ['a'])])).toEqual([]); }); - - it('skips tags that connect too many notes', () => { - const notes = Array.from({ length: 21 }, (_, i) => - note(`n${i}`, `N${i}`, [], ['common']) - ); - - expect(factory.createEdges(notes)).toEqual([]); - }); }); From 804c991cb80b9b22e03b7366df17ba7d33d12bc1 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Sun, 12 Jul 2026 15:24:57 +0530 Subject: [PATCH 14/28] ANG-007: update package-lock --- package-lock.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index f2a3a72..396cf14 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,7 @@ }, "devDependencies": { "@types/jest": "^29.5.14", - "@types/node": "^18.7.13", + "@types/node": "^18.19.130", "chalk": "^4.1.0", "copy-webpack-plugin": "^11.0.0", "fs-extra": "^10.1.0", From 9e569f3f9d92159e422d072180ce939e7b08abff Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Sun, 12 Jul 2026 18:54:18 +0530 Subject: [PATCH 15/28] ANG-007: some bug fixes --- src/services/embeddings/Orchestrator.ts | 30 ++++++++----------- .../embeddings/ProviderResolver.test.ts | 3 ++ src/services/embeddings/ProviderResolver.ts | 6 ++-- .../Providers/JoplinNativeProvider.ts | 15 +++++----- src/ui/graph-view.js | 7 ++++- 5 files changed, 32 insertions(+), 29 deletions(-) diff --git a/src/services/embeddings/Orchestrator.ts b/src/services/embeddings/Orchestrator.ts index 64e5661..2a1d3be 100644 --- a/src/services/embeddings/Orchestrator.ts +++ b/src/services/embeddings/Orchestrator.ts @@ -24,28 +24,25 @@ export class EmbeddingOrchestrator { this.cancelled = true; } - /** - * Embeds the provided notes, preserving note order and reporting missing - * embeddings as per-note errors. - */ public async embedNotes(notes: Note[]): Promise { - const embeddedNotes: EmbeddedNote[] = []; - const errors: Array<{ noteId: string; error: string }> = []; - if (!notes || notes.length === 0) { - return { embeddedNotes, errors }; + return { embeddedNotes: [], errors: [] }; } if (!this.provider) { - return { embeddedNotes, errors: notes.map(function (n) { return { noteId: n.id, error: 'No provider configured' }; }) }; + const errors = notes.map(n => ({ noteId: n.id, error: 'No provider configured' })); + return { embeddedNotes: [], errors }; } try { this.reportProgress(0, notes.length, 'embedding'); - const noteIds = notes.map(function (n) { return n.id; }); + const noteIds = notes.map(n => n.id); const vectorsByNoteId = await this.provider.fetchVectorsByNoteIds(noteIds); + const embeddedNotes: EmbeddedNote[] = []; + const errors: Array<{ noteId: string; error: string }> = []; + for (let i = 0; i < notes.length; i++) { if (this.cancelled) break; const note = notes[i]; @@ -53,20 +50,17 @@ export class EmbeddingOrchestrator { if (vector) { embeddedNotes.push({ note: note, embedding: vector }); } else { - errors.push({ noteId: note.id, error: 'Note not yet indexed by Joplin AI. Wait for indexing to complete.' }); + errors.push({ noteId: note.id, error: 'Note not yet indexed by Joplin AI.' }); } this.reportProgress(i + 1, notes.length, 'embedding'); } + + return { embeddedNotes, errors }; } catch (e) { const msg = e instanceof Error ? e.message : String(e); - for (let n = 0; n < notes.length; n++) { - if (!embeddedNotes.some(function (en) { return en.note.id === notes[n].id; })) { - errors.push({ noteId: notes[n].id, error: msg }); - } - } + const errors = notes.map(n => ({ noteId: n.id, error: msg })); + return { embeddedNotes: [], errors }; } - - return { embeddedNotes, errors }; } private reportProgress(current: number, total: number, phase: BatchProgress['phase']): void { diff --git a/src/services/embeddings/ProviderResolver.test.ts b/src/services/embeddings/ProviderResolver.test.ts index 242af28..6afe48e 100644 --- a/src/services/embeddings/ProviderResolver.test.ts +++ b/src/services/embeddings/ProviderResolver.test.ts @@ -29,6 +29,7 @@ describe('ProviderResolver', () => { it('throws when index is not ready', async () => { (joplin as any).ai = { getIndexStatus: jest.fn().mockResolvedValue({ ready: false, state: 'disabled' }), + getEmbeddings: jest.fn(), }; await expect(ProviderResolver.resolveWithValidation()).rejects.toThrow( 'Joplin AI index is not ready' @@ -38,6 +39,7 @@ describe('ProviderResolver', () => { it('returns provider when index is ready', async () => { (joplin as any).ai = { getIndexStatus: jest.fn().mockResolvedValue({ ready: true, modelId: 'test-model' }), + getEmbeddings: jest.fn(), }; const provider = await ProviderResolver.resolveWithValidation(); expect(provider.id).toBe('joplin-native'); @@ -47,6 +49,7 @@ describe('ProviderResolver', () => { it('uses the default native model when index status omits modelId', async () => { (joplin as any).ai = { getIndexStatus: jest.fn().mockResolvedValue({ ready: true }), + getEmbeddings: jest.fn(), }; const provider = await ProviderResolver.resolveWithValidation(); expect(provider.modelName).toBe('joplin-native'); diff --git a/src/services/embeddings/ProviderResolver.ts b/src/services/embeddings/ProviderResolver.ts index 0c864a3..0bc9979 100644 --- a/src/services/embeddings/ProviderResolver.ts +++ b/src/services/embeddings/ProviderResolver.ts @@ -1,6 +1,6 @@ import joplin from 'api'; import { EmbeddingProvider, ProviderConfig } from './Types'; -import { JoplinNativeProvider, JoplinAiApi } from './Providers/JoplinNativeProvider'; +import { JoplinNativeProvider, JoplinAiApi } from './providers/JoplinNativeProvider'; export class ProviderResolver { @@ -14,14 +14,14 @@ export class ProviderResolver { */ public static async resolveWithValidation(): Promise { const joplinAi = joplin.ai as unknown as JoplinAiApi | undefined; - if (!joplinAi || typeof joplinAi.getIndexStatus !== 'function') { + if (!joplinAi || typeof joplinAi.getIndexStatus !== 'function' || typeof joplinAi.getEmbeddings !== 'function') { throw new Error('joplin.ai is not available. Enable AI in Settings → AI. Requires Joplin v3.7+.'); } const status = await joplinAi.getIndexStatus(); if (!status || !status.ready) { throw new Error('Joplin AI index is not ready. Enable AI and the embedding index in Settings → AI.'); } - return new JoplinNativeProvider(status.modelId ?? JoplinNativeProvider.DEFAULT_MODEL_ID, 0); + return new JoplinNativeProvider(status.modelId ?? JoplinNativeProvider.DEFAULT_MODEL_ID); } public static getDefaultConfig(): ProviderConfig { diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.ts b/src/services/embeddings/Providers/JoplinNativeProvider.ts index 505018b..a2873ed 100644 --- a/src/services/embeddings/Providers/JoplinNativeProvider.ts +++ b/src/services/embeddings/Providers/JoplinNativeProvider.ts @@ -19,15 +19,14 @@ export class JoplinNativeProvider implements EmbeddingProvider { public readonly id: ProviderId = 'joplin-native'; public static readonly DEFAULT_MODEL_ID = 'joplin-native'; private static readonly PAGE_SIZE = 1000; + private static readonly MAX_PAGES = 500; private _modelName: string; - private _dimension: number; private cachedVectors: Map | null = null; private fetchedModelId: string | null = null; - public constructor(modelName: string = JoplinNativeProvider.DEFAULT_MODEL_ID, dimension: number = 0) { + public constructor(modelName: string = JoplinNativeProvider.DEFAULT_MODEL_ID) { this._modelName = modelName; - this._dimension = dimension; } public get modelName(): string { @@ -92,9 +91,15 @@ export class JoplinNativeProvider implements EmbeddingProvider { const grouped = new Map(); let cursor: string | undefined; let modelChangeRetries = 0; + let pageCount = 0; const MAX_MODEL_CHANGE_RETRIES = 3; while (true) { + if (pageCount >= JoplinNativeProvider.MAX_PAGES) { + throw new Error('Too many pages. The embedding index may be in an unexpected state.'); + } + pageCount++; + const page = await api.getEmbeddings({ noteIds: noteIds, cursor: cursor, @@ -118,10 +123,6 @@ export class JoplinNativeProvider implements EmbeddingProvider { } } - if (this._dimension === 0 && page.dimension > 0) { - this._dimension = page.dimension; - } - for (const chunk of page.chunks) { const list = grouped.get(chunk.noteId); if (list) { diff --git a/src/ui/graph-view.js b/src/ui/graph-view.js index ac6e14c..4fa389c 100644 --- a/src/ui/graph-view.js +++ b/src/ui/graph-view.js @@ -204,7 +204,12 @@ function renderGraph(message) { cy.layout(FCOSE_OPTIONS).run(); - hideStatus(); + var edgeCount = (message.edges || []).length; + if (edgeCount === 0) { + showStatus(message.nodes.length + ' notes, 0 connections'); + } else { + hideStatus(); + } } /** Write counts into the stats bar elements (stat-notes, stat-explicit, stat-semantic, stat-tags). */ From 626f665fce5deed4f59b101f248762e81a2f2b41 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Sun, 12 Jul 2026 19:10:07 +0530 Subject: [PATCH 16/28] ANG-007: removed dead code --- src/services/embeddings/Orchestrator.test.ts | 6 +++--- src/services/embeddings/Orchestrator.ts | 8 ++++---- src/services/embeddings/Types.ts | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/services/embeddings/Orchestrator.test.ts b/src/services/embeddings/Orchestrator.test.ts index a8652b0..d89e820 100644 --- a/src/services/embeddings/Orchestrator.test.ts +++ b/src/services/embeddings/Orchestrator.test.ts @@ -95,9 +95,9 @@ describe('EmbeddingOrchestrator', () => { ]); expect(progressUpdates).toEqual([ - { current: 0, total: 2, phase: 'embedding' }, - { current: 1, total: 2, phase: 'embedding' }, - { current: 2, total: 2, phase: 'embedding' }, + { current: 0, total: 2 }, + { current: 1, total: 2 }, + { current: 2, total: 2 }, ]); }); diff --git a/src/services/embeddings/Orchestrator.ts b/src/services/embeddings/Orchestrator.ts index 2a1d3be..896f736 100644 --- a/src/services/embeddings/Orchestrator.ts +++ b/src/services/embeddings/Orchestrator.ts @@ -35,7 +35,7 @@ export class EmbeddingOrchestrator { } try { - this.reportProgress(0, notes.length, 'embedding'); + this.reportProgress(0, notes.length); const noteIds = notes.map(n => n.id); const vectorsByNoteId = await this.provider.fetchVectorsByNoteIds(noteIds); @@ -52,7 +52,7 @@ export class EmbeddingOrchestrator { } else { errors.push({ noteId: note.id, error: 'Note not yet indexed by Joplin AI.' }); } - this.reportProgress(i + 1, notes.length, 'embedding'); + this.reportProgress(i + 1, notes.length); } return { embeddedNotes, errors }; @@ -63,9 +63,9 @@ export class EmbeddingOrchestrator { } } - private reportProgress(current: number, total: number, phase: BatchProgress['phase']): void { + private reportProgress(current: number, total: number): void { if (this.onProgress) { - this.onProgress({ current, total, phase }); + this.onProgress({ current, total }); } } } diff --git a/src/services/embeddings/Types.ts b/src/services/embeddings/Types.ts index bb98018..864b8af 100644 --- a/src/services/embeddings/Types.ts +++ b/src/services/embeddings/Types.ts @@ -22,7 +22,6 @@ export interface EmbeddedNote { export interface BatchProgress { current: number; total: number; - phase: 'preprocessing' | 'embedding'; } export interface EmbeddingResult { From 41bf0a3cf7af3a13d4910c1dd0bf32aa24f03b85 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Mon, 13 Jul 2026 12:26:41 +0530 Subject: [PATCH 17/28] ANG-007: added suggested fixes --- src/services/embeddings/ProviderResolver.test.ts | 9 --------- src/services/embeddings/ProviderResolver.ts | 6 +----- .../embeddings/Providers/JoplinNativeProvider.ts | 4 ++-- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/src/services/embeddings/ProviderResolver.test.ts b/src/services/embeddings/ProviderResolver.test.ts index 6afe48e..e8b4072 100644 --- a/src/services/embeddings/ProviderResolver.test.ts +++ b/src/services/embeddings/ProviderResolver.test.ts @@ -2,15 +2,6 @@ import { ProviderResolver } from './ProviderResolver'; import joplin from 'api'; describe('ProviderResolver', () => { - describe('resolve', () => { - it('returns JoplinNativeProvider', () => { - const provider = ProviderResolver.resolve(); - expect(provider).toBeDefined(); - expect(provider.id).toBe('joplin-native'); - expect(provider.modelName).toBe('joplin-native'); - }); - }); - describe('getDefaultConfig', () => { it('returns joplin-native as default', () => { const config = ProviderResolver.getDefaultConfig(); diff --git a/src/services/embeddings/ProviderResolver.ts b/src/services/embeddings/ProviderResolver.ts index 0bc9979..842b450 100644 --- a/src/services/embeddings/ProviderResolver.ts +++ b/src/services/embeddings/ProviderResolver.ts @@ -4,10 +4,6 @@ import { JoplinNativeProvider, JoplinAiApi } from './providers/JoplinNativeProvi export class ProviderResolver { - public static resolve(): EmbeddingProvider { - return new JoplinNativeProvider(); - } - /** * Resolves the native embedding provider after verifying that Joplin AI is * available and its embedding index is ready. @@ -25,6 +21,6 @@ export class ProviderResolver { } public static getDefaultConfig(): ProviderConfig { - return { id: 'joplin-native' }; + return { id: JoplinNativeProvider.DEFAULT_MODEL_ID }; } } diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.ts b/src/services/embeddings/Providers/JoplinNativeProvider.ts index a2873ed..ed6a47a 100644 --- a/src/services/embeddings/Providers/JoplinNativeProvider.ts +++ b/src/services/embeddings/Providers/JoplinNativeProvider.ts @@ -20,6 +20,7 @@ export class JoplinNativeProvider implements EmbeddingProvider { public static readonly DEFAULT_MODEL_ID = 'joplin-native'; private static readonly PAGE_SIZE = 1000; private static readonly MAX_PAGES = 500; + private static readonly MAX_MODEL_CHANGE_RETRIES = 3; private _modelName: string; private cachedVectors: Map | null = null; @@ -92,7 +93,6 @@ export class JoplinNativeProvider implements EmbeddingProvider { let cursor: string | undefined; let modelChangeRetries = 0; let pageCount = 0; - const MAX_MODEL_CHANGE_RETRIES = 3; while (true) { if (pageCount >= JoplinNativeProvider.MAX_PAGES) { @@ -113,7 +113,7 @@ export class JoplinNativeProvider implements EmbeddingProvider { trackedModelId = pageModelId; } else if (pageModelId !== trackedModelId) { modelChangeRetries++; - if (modelChangeRetries > MAX_MODEL_CHANGE_RETRIES) { + if (modelChangeRetries > JoplinNativeProvider.MAX_MODEL_CHANGE_RETRIES) { throw new Error('Model changed too many times during pagination.'); } trackedModelId = pageModelId; From c14f42ec377d207389dc7e843cf270235117bba6 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Mon, 13 Jul 2026 22:40:10 +0530 Subject: [PATCH 18/28] ANG-008:Similarity computation, edge creation & SQLite cache --- src/data/Database/VectorDatabase.ts | 78 +++ src/data/Database/VectorRepository.test.ts | 125 +++++ src/data/Database/VectorRepository.ts | 121 ++++ src/services/embeddings/Orchestrator.test.ts | 157 +++++- src/services/embeddings/Orchestrator.ts | 86 ++- .../embeddings/ProviderResolver.test.ts | 27 +- src/services/embeddings/ProviderResolver.ts | 13 +- .../Providers/JoplinNativeProvider.test.ts | 42 +- .../Providers/JoplinNativeProvider.ts | 113 ++-- src/services/graph/GraphBuilder.test.ts | 44 ++ src/services/graph/GraphBuilder.ts | 65 ++- src/services/similarity/EdgeFactory.test.ts | 21 + src/services/similarity/EdgeFactory.ts | 58 +- .../similarity/SimilarityEngine.test.ts | 515 ++++++++++++++++++ src/services/similarity/SimilarityEngine.ts | 316 +++++++++++ src/services/similarity/ThresholdPresets.ts | 26 + 16 files changed, 1738 insertions(+), 69 deletions(-) create mode 100644 src/data/Database/VectorDatabase.ts create mode 100644 src/data/Database/VectorRepository.test.ts create mode 100644 src/data/Database/VectorRepository.ts create mode 100644 src/services/similarity/SimilarityEngine.test.ts create mode 100644 src/services/similarity/SimilarityEngine.ts create mode 100644 src/services/similarity/ThresholdPresets.ts diff --git a/src/data/Database/VectorDatabase.ts b/src/data/Database/VectorDatabase.ts new file mode 100644 index 0000000..c6780f4 --- /dev/null +++ b/src/data/Database/VectorDatabase.ts @@ -0,0 +1,78 @@ +import joplin from 'api'; + +export interface IVectorDatabase { + open(): Promise; + run(sql: string, params: unknown[]): Promise; + all(sql: string, params: unknown[]): Promise; +} + +interface Sqlite3Database { + run(sql: string, params: unknown[], callback: (err: Error | null) => void): void; + all(sql: string, params: unknown[], callback: (err: Error | null, rows: unknown[]) => void): void; +} + +/** + * Thin promisified wrapper around Joplin's bundled sqlite3 module (accessed via + * `joplin.require('sqlite3')`, since native packages can't be bundled with a + * plugin). Owns only the connection and schema; query logic lives in + * VectorRepository. + */ +export class VectorDatabase implements IVectorDatabase { + private static readonly DB_FILE_NAME = 'note-graph-vectors.sqlite'; + private static readonly SCHEMA = ` + CREATE TABLE IF NOT EXISTS note_vectors ( + note_id TEXT PRIMARY KEY, + model_id TEXT NOT NULL, + updated_time INTEGER NOT NULL, + vector BLOB NOT NULL + ) + `; + + private db: Sqlite3Database | null = null; + private opening: Promise | null = null; + + /** Opens (creating if needed) the vector cache database. Safe to call repeatedly. */ + public async open(): Promise { + if (this.db) return; + if (!this.opening) { + this.opening = this.openInternal(); + } + await this.opening; + } + + public async run(sql: string, params: unknown[]): Promise { + const db = this.requireDb(); + await new Promise((resolve, reject) => { + db.run(sql, params, (err) => (err ? reject(err) : resolve())); + }); + } + + public async all(sql: string, params: unknown[]): Promise { + const db = this.requireDb(); + return new Promise((resolve, reject) => { + db.all(sql, params, (err, rows) => (err ? reject(err) : resolve(rows as T[]))); + }); + } + + private async openInternal(): Promise { + const sqlite3 = joplin.require('sqlite3'); + const dataDir = await joplin.plugins.dataDir(); + const dbPath = `${dataDir}/${VectorDatabase.DB_FILE_NAME}`; + + this.db = await new Promise((resolve, reject) => { + const db = new sqlite3.Database(dbPath, (err: Error | null) => { + if (err) reject(err); + else resolve(db); + }); + }); + + await this.run(VectorDatabase.SCHEMA, []); + } + + private requireDb(): Sqlite3Database { + if (!this.db) { + throw new Error('VectorDatabase used before open() completed.'); + } + return this.db; + } +} diff --git a/src/data/Database/VectorRepository.test.ts b/src/data/Database/VectorRepository.test.ts new file mode 100644 index 0000000..4b50115 --- /dev/null +++ b/src/data/Database/VectorRepository.test.ts @@ -0,0 +1,125 @@ +import { VectorRepository } from './VectorRepository'; +import { IVectorDatabase } from './VectorDatabase'; + +/** + * In-memory stand-in for VectorDatabase. sqlite3 is only reachable at runtime + * via joplin.require(), so VectorRepository is tested against this fake + * rather than a real database; it emulates the single upsert statement and + * the `note_id IN (...)` select that VectorRepository issues. + */ +class FakeVectorDatabase implements IVectorDatabase { + public opened = false; + public allCallBatchSizes: number[] = []; + private rows = new Map(); + + public async open(): Promise { + this.opened = true; + } + + public async run(_sql: string, params: unknown[]): Promise { + if (params.length === 0) { + return; // BEGIN TRANSACTION / COMMIT / ROLLBACK + } + const [noteId, modelId, updatedTime, vector] = params as [string, string, number, Buffer]; + this.rows.set(noteId, { note_id: noteId, model_id: modelId, updated_time: updatedTime, vector }); + } + + public async all(_sql: string, params: unknown[]): Promise { + const ids = params as string[]; + this.allCallBatchSizes.push(ids.length); + const found = ids.map(id => this.rows.get(id)).filter((r): r is NonNullable => !!r); + return found as unknown as T[]; + } +} + +describe('VectorRepository', () => { + let db: FakeVectorDatabase; + let repo: VectorRepository; + + beforeEach(() => { + db = new FakeVectorDatabase(); + repo = new VectorRepository(db); + }); + + describe('getMany', () => { + it('returns an empty map without opening the database for no IDs', async () => { + const result = await repo.getMany([]); + expect(result.size).toBe(0); + expect(db.opened).toBe(false); + }); + + it('returns nothing for IDs that were never saved', async () => { + const result = await repo.getMany(['missing']); + expect(result.size).toBe(0); + expect(db.opened).toBe(true); + }); + }); + + describe('saveMany + getMany round trip', () => { + it('round-trips vector values through the Float32 BLOB encoding', async () => { + const vector = [0.1, -0.25, 0.987654, 1, -1, 0]; + await repo.saveMany([{ noteId: 'n1', vector, modelId: 'm1', updatedTime: 100 }]); + + const result = await repo.getMany(['n1']); + const entry = result.get('n1'); + + expect(entry).toBeDefined(); + expect(entry!.modelId).toBe('m1'); + expect(entry!.updatedTime).toBe(100); + expect(entry!.vector).toHaveLength(vector.length); + for (let i = 0; i < vector.length; i++) { + // Float32 storage loses some precision relative to the JS float64 input. + expect(entry!.vector[i]).toBeCloseTo(vector[i], 5); + } + }); + + it('overwrites the previous entry for the same note ID', async () => { + await repo.saveMany([{ noteId: 'n1', vector: [1, 0], modelId: 'm1', updatedTime: 100 }]); + await repo.saveMany([{ noteId: 'n1', vector: [0, 1], modelId: 'm2', updatedTime: 200 }]); + + const result = await repo.getMany(['n1']); + const entry = result.get('n1'); + + expect(entry!.modelId).toBe('m2'); + expect(entry!.updatedTime).toBe(200); + expect(entry!.vector[0]).toBeCloseTo(0, 5); + expect(entry!.vector[1]).toBeCloseTo(1, 5); + }); + + it('only returns entries for the requested IDs that exist', async () => { + await repo.saveMany([ + { noteId: 'n1', vector: [1, 0], modelId: 'm1', updatedTime: 100 }, + { noteId: 'n2', vector: [0, 1], modelId: 'm1', updatedTime: 100 }, + ]); + + const result = await repo.getMany(['n1', 'n3']); + + expect(result.has('n1')).toBe(true); + expect(result.has('n2')).toBe(false); + expect(result.has('n3')).toBe(false); + }); + + it('does nothing for an empty entries array', async () => { + await repo.saveMany([]); + expect(db.opened).toBe(false); + }); + }); + + describe('large vaults', () => { + it('chunks getMany so a single query never exceeds SQLite\'s bound-parameter limit', async () => { + const noteIds = Array.from({ length: 1200 }, (_, i) => `n${i}`); + await repo.saveMany( + noteIds.map(id => ({ noteId: id, vector: [1, 0], modelId: 'm1', updatedTime: 1 })), + ); + + const result = await repo.getMany(noteIds); + + expect(result.size).toBe(1200); + expect(db.allCallBatchSizes.length).toBeGreaterThan(1); + for (const size of db.allCallBatchSizes) { + expect(size).toBeLessThanOrEqual(500); + } + expect(db.allCallBatchSizes.reduce((a, b) => a + b, 0)).toBe(1200); + }); + }); +}); diff --git a/src/data/Database/VectorRepository.ts b/src/data/Database/VectorRepository.ts new file mode 100644 index 0000000..4757521 --- /dev/null +++ b/src/data/Database/VectorRepository.ts @@ -0,0 +1,121 @@ +import { IVectorDatabase, VectorDatabase } from './VectorDatabase'; + +export interface CachedVector { + vector: number[]; + modelId: string; + updatedTime: number; +} + +export interface VectorCacheEntry { + noteId: string; + vector: number[]; + modelId: string; + updatedTime: number; +} + +export interface VectorCache { + getMany(noteIds: string[]): Promise>; + saveMany(entries: VectorCacheEntry[]): Promise; +} + +interface VectorRow { + note_id: string; + model_id: string; + updated_time: number; + vector: Buffer; +} + +/** + * Persists note embedding vectors in SQLite so unchanged notes aren't + * re-fetched from joplin.ai.getEmbeddings(). A cached vector is only reused + * when both its note_id and model_id match, since staleness is decided by + * the caller comparing `updatedTime` against the note's current updated_time. + */ +export class VectorRepository implements VectorCache { + /** SQLite caps bound parameters per statement (as low as 999 on some builds); stay well under it. */ + private static readonly QUERY_BATCH_SIZE = 500; + + public constructor(private readonly db: IVectorDatabase = new VectorDatabase()) {} + + /** Returns cached vectors for the given note IDs, keyed by note ID. Missing notes are omitted. */ + public async getMany(noteIds: string[]): Promise> { + if (noteIds.length === 0) { + return new Map(); + } + + await this.db.open(); + + const result = new Map(); + for (const batch of this.chunk(noteIds, VectorRepository.QUERY_BATCH_SIZE)) { + const rows = await this.queryBatch(batch); + for (const row of rows) { + result.set(row.note_id, { + vector: this.decodeVector(row.vector), + modelId: row.model_id, + updatedTime: row.updated_time, + }); + } + } + return result; + } + + /** Inserts or updates vectors for the given notes, in a single transaction. */ + public async saveMany(entries: VectorCacheEntry[]): Promise { + if (entries.length === 0) { + return; + } + + await this.db.open(); + + await this.db.run('BEGIN TRANSACTION', []); + try { + for (const entry of entries) { + await this.db.run( + `INSERT INTO note_vectors (note_id, model_id, updated_time, vector) + VALUES (?, ?, ?, ?) + ON CONFLICT(note_id) DO UPDATE SET + model_id = excluded.model_id, + updated_time = excluded.updated_time, + vector = excluded.vector`, + [entry.noteId, entry.modelId, entry.updatedTime, this.encodeVector(entry.vector)], + ); + } + await this.db.run('COMMIT', []); + } catch (e) { + await this.db.run('ROLLBACK', []); + throw e; + } + } + + private async queryBatch(noteIds: string[]): Promise { + const placeholders = noteIds.map(() => '?').join(','); + return this.db.all( + `SELECT note_id, model_id, updated_time, vector FROM note_vectors WHERE note_id IN (${placeholders})`, + noteIds, + ); + } + + private chunk(items: T[], size: number): T[][] { + const batches: T[][] = []; + for (let i = 0; i < items.length; i += size) { + batches.push(items.slice(i, i + size)); + } + return batches; + } + + /** Encodes a vector as a Float32 BLOB for compact SQLite storage. */ + private encodeVector(vector: number[]): Buffer { + const floats = Float32Array.from(vector); + return Buffer.from(floats.buffer, floats.byteOffset, floats.byteLength); + } + + /** Decodes a Float32 BLOB back into a plain number array. */ + private decodeVector(blob: Buffer): number[] { + const floats = new Float32Array( + blob.buffer, + blob.byteOffset, + blob.byteLength / Float32Array.BYTES_PER_ELEMENT, + ); + return Array.from(floats); + } +} diff --git a/src/services/embeddings/Orchestrator.test.ts b/src/services/embeddings/Orchestrator.test.ts index d89e820..366d4c6 100644 --- a/src/services/embeddings/Orchestrator.test.ts +++ b/src/services/embeddings/Orchestrator.test.ts @@ -2,14 +2,14 @@ import { EmbeddingOrchestrator } from './Orchestrator'; import { BatchProgress } from './Types'; import { Note } from '../../data/Types'; -function makeNote(id: string, title: string, body: string): Note { +function makeNote(id: string, title: string, body: string, updatedTime = 0): Note { return { id, parent_id: 'p1', title, body, created_time: 0, - updated_time: 0, + updated_time: updatedTime, }; } @@ -128,4 +128,157 @@ describe('EmbeddingOrchestrator', () => { expect(result.errors[0].error).toBe('API failure'); }); }); + + describe('vector caching', () => { + it('serves an unchanged, same-model note from the cache without calling the provider', async () => { + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map()); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setCache({ + getMany: jest.fn().mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]), + ), + saveMany: jest.fn().mockResolvedValue(undefined), + }); + + const result = await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 50)]); + + expect(fetchVectorsByNoteIds).not.toHaveBeenCalled(); + expect(result.embeddedNotes).toHaveLength(1); + expect(result.embeddedNotes[0].embedding).toEqual([0.1, 0.2]); + }); + + it('re-fetches a note whose updated_time no longer matches the cached entry', async () => { + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.9, 0.9]]])); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setCache({ + getMany: jest.fn().mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]), + ), + saveMany: jest.fn().mockResolvedValue(undefined), + }); + + const result = await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 999)]); + + expect(fetchVectorsByNoteIds).toHaveBeenCalledWith(['n1']); + expect(result.embeddedNotes[0].embedding).toEqual([0.9, 0.9]); + }); + + it('does not fall back to a stale cached vector when the re-fetch omits the note', async () => { + // The note changed (updated_time no longer matches), so it's correctly + // queued for re-fetch — but Joplin's AI index hasn't caught up yet and + // returns nothing for it. The stale pre-edit vector must not be used. + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map()); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setCache({ + getMany: jest.fn().mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]), + ), + saveMany: jest.fn().mockResolvedValue(undefined), + }); + + const result = await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 999)]); + + expect(fetchVectorsByNoteIds).toHaveBeenCalledWith(['n1']); + expect(result.embeddedNotes).toHaveLength(0); + expect(result.errors).toEqual([{ noteId: 'n1', error: 'Note not yet indexed by Joplin AI.' }]); + }); + + it('re-fetches a note whose cached entry belongs to a different embedding model', async () => { + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.9, 0.9]]])); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm2', fetchVectorsByNoteIds }); + orchestrator.setCache({ + getMany: jest.fn().mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]), + ), + saveMany: jest.fn().mockResolvedValue(undefined), + }); + + const result = await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 50)]); + + expect(fetchVectorsByNoteIds).toHaveBeenCalledWith(['n1']); + expect(result.embeddedNotes[0].embedding).toEqual([0.9, 0.9]); + }); + + it('only asks the provider for stale/missing notes, mixing in cache hits', async () => { + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n2', [0.4, 0.5]]])); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setCache({ + getMany: jest.fn().mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 10 }]]), + ), + saveMany: jest.fn().mockResolvedValue(undefined), + }); + + const result = await orchestrator.embedNotes([ + makeNote('n1', 'T1', 'B1', 10), + makeNote('n2', 'T2', 'B2', 20), + ]); + + expect(fetchVectorsByNoteIds).toHaveBeenCalledWith(['n2']); + expect(result.embeddedNotes).toHaveLength(2); + expect(result.errors).toHaveLength(0); + }); + + it('saves freshly fetched vectors back to the cache with the note updated_time and model', async () => { + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); + const saveMany = jest.fn().mockResolvedValue(undefined); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setCache({ getMany: jest.fn().mockResolvedValue(new Map()), saveMany }); + + await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 42)]); + + expect(saveMany).toHaveBeenCalledWith([ + { noteId: 'n1', vector: [0.4, 0.5], modelId: 'm1', updatedTime: 42 }, + ]); + }); + + it('does not save notes the provider failed to return', async () => { + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map()); + const saveMany = jest.fn().mockResolvedValue(undefined); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setCache({ getMany: jest.fn().mockResolvedValue(new Map()), saveMany }); + + await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 42)]); + + expect(saveMany).not.toHaveBeenCalled(); + }); + + it('falls back to a full fetch when the cache read throws', async () => { + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setCache({ + getMany: jest.fn().mockRejectedValue(new Error('disk error')), + saveMany: jest.fn().mockResolvedValue(undefined), + }); + + const result = await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 42)]); + + expect(fetchVectorsByNoteIds).toHaveBeenCalledWith(['n1']); + expect(result.embeddedNotes[0].embedding).toEqual([0.4, 0.5]); + }); + + it('still returns results when the cache write throws', async () => { + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setCache({ + getMany: jest.fn().mockResolvedValue(new Map()), + saveMany: jest.fn().mockRejectedValue(new Error('disk full')), + }); + + const result = await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 42)]); + + expect(result.embeddedNotes[0].embedding).toEqual([0.4, 0.5]); + expect(result.errors).toHaveLength(0); + }); + + it('behaves exactly as before when no cache is set', async () => { + const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); + orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + + const result = await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 42)]); + + expect(fetchVectorsByNoteIds).toHaveBeenCalledWith(['n1']); + expect(result.embeddedNotes[0].embedding).toEqual([0.4, 0.5]); + }); + }); }); diff --git a/src/services/embeddings/Orchestrator.ts b/src/services/embeddings/Orchestrator.ts index 896f736..130ff9a 100644 --- a/src/services/embeddings/Orchestrator.ts +++ b/src/services/embeddings/Orchestrator.ts @@ -1,4 +1,5 @@ import { Note } from '../../data/Types'; +import { CachedVector, VectorCache, VectorCacheEntry } from '../../data/Database/VectorRepository'; import { EmbeddingProvider, EmbeddedNote, @@ -8,6 +9,7 @@ import { export class EmbeddingOrchestrator { private provider: EmbeddingProvider | null = null; + private cache: VectorCache | null = null; private cancelled: boolean = false; private onProgress: ((progress: BatchProgress) => void) | null = null; @@ -16,6 +18,11 @@ export class EmbeddingOrchestrator { this.cancelled = false; } + /** Injects a persistent vector cache so unchanged notes skip re-fetching. Optional. */ + public setCache(cache: VectorCache): void { + this.cache = cache; + } + public setOnProgress(callback: (progress: BatchProgress) => void): void { this.onProgress = callback; } @@ -37,8 +44,7 @@ export class EmbeddingOrchestrator { try { this.reportProgress(0, notes.length); - const noteIds = notes.map(n => n.id); - const vectorsByNoteId = await this.provider.fetchVectorsByNoteIds(noteIds); + const vectorsByNoteId = await this.resolveVectors(notes, this.provider); const embeddedNotes: EmbeddedNote[] = []; const errors: Array<{ noteId: string; error: string }> = []; @@ -63,6 +69,82 @@ export class EmbeddingOrchestrator { } } + /** + * Resolves a vector per note, reusing cached vectors for notes whose + * `updated_time` and model haven't changed and only asking the provider + * to (re-)fetch the rest. + */ + private async resolveVectors(notes: Note[], provider: EmbeddingProvider): Promise> { + const modelId = provider.modelName; + const cached = await this.getCachedVectors(notes); + + const notesToFetch = notes.filter(n => !this.isFreshCacheHit(cached.get(n.id), n, modelId)); + + const fresh = notesToFetch.length > 0 + ? await provider.fetchVectorsByNoteIds(notesToFetch.map(n => n.id)) + : new Map(); + + await this.saveFreshVectors(notesToFetch, fresh, modelId); + + return this.mergeVectors(notes, cached, fresh, modelId); + } + + /** Combines still-fresh cached vectors with newly fetched ones, keyed by note ID. */ + private mergeVectors( + notes: Note[], + cached: Map, + fresh: Map, + modelId: string, + ): Map { + const merged = new Map(); + for (const note of notes) { + const entry = cached.get(note.id); + if (entry && this.isFreshCacheHit(entry, note, modelId)) { + merged.set(note.id, entry.vector); + } + } + for (const [noteId, vector] of fresh) merged.set(noteId, vector); + return merged; + } + + /** A cache entry is only reusable if the note is unchanged and the embedding model hasn't changed. */ + private isFreshCacheHit(entry: CachedVector | undefined, note: Note, modelId: string): boolean { + return !!entry && entry.updatedTime === note.updated_time && entry.modelId === modelId; + } + + private async getCachedVectors(notes: Note[]): Promise> { + if (!this.cache) return new Map(); + try { + return await this.cache.getMany(notes.map(n => n.id)); + } catch (e) { + console.error('Vector cache read failed, falling back to a full fetch:', e); + return new Map(); + } + } + + private async saveFreshVectors( + notes: Note[], + vectors: Map, + modelId: string, + ): Promise { + if (!this.cache || vectors.size === 0) return; + + const entries: VectorCacheEntry[] = notes + .filter(n => vectors.has(n.id)) + .map(n => ({ + noteId: n.id, + vector: vectors.get(n.id)!, + modelId, + updatedTime: n.updated_time, + })); + + try { + await this.cache.saveMany(entries); + } catch (e) { + console.error('Vector cache write failed:', e); + } + } + private reportProgress(current: number, total: number): void { if (this.onProgress) { this.onProgress({ current, total }); diff --git a/src/services/embeddings/ProviderResolver.test.ts b/src/services/embeddings/ProviderResolver.test.ts index e8b4072..9d9ad58 100644 --- a/src/services/embeddings/ProviderResolver.test.ts +++ b/src/services/embeddings/ProviderResolver.test.ts @@ -17,19 +17,29 @@ describe('ProviderResolver', () => { ); }); - it('throws when index is not ready', async () => { + it('throws when index is disabled', async () => { (joplin as any).ai = { getIndexStatus: jest.fn().mockResolvedValue({ ready: false, state: 'disabled' }), getEmbeddings: jest.fn(), }; await expect(ProviderResolver.resolveWithValidation()).rejects.toThrow( - 'Joplin AI index is not ready' + 'Joplin AI index is not usable yet (state: disabled)' + ); + }); + + it('throws while the embedding model is still preparing', async () => { + (joplin as any).ai = { + getIndexStatus: jest.fn().mockResolvedValue({ ready: false, state: 'preparing' }), + getEmbeddings: jest.fn(), + }; + await expect(ProviderResolver.resolveWithValidation()).rejects.toThrow( + 'Joplin AI index is not usable yet (state: preparing)' ); }); it('returns provider when index is ready', async () => { (joplin as any).ai = { - getIndexStatus: jest.fn().mockResolvedValue({ ready: true, modelId: 'test-model' }), + getIndexStatus: jest.fn().mockResolvedValue({ ready: true, state: 'ready', modelId: 'test-model' }), getEmbeddings: jest.fn(), }; const provider = await ProviderResolver.resolveWithValidation(); @@ -37,9 +47,18 @@ describe('ProviderResolver', () => { expect(provider.modelName).toBe('test-model'); }); + it('returns provider while the index is still indexing, since search still works with partial data', async () => { + (joplin as any).ai = { + getIndexStatus: jest.fn().mockResolvedValue({ ready: false, state: 'indexing', modelId: 'test-model' }), + getEmbeddings: jest.fn(), + }; + const provider = await ProviderResolver.resolveWithValidation(); + expect(provider.modelName).toBe('test-model'); + }); + it('uses the default native model when index status omits modelId', async () => { (joplin as any).ai = { - getIndexStatus: jest.fn().mockResolvedValue({ ready: true }), + getIndexStatus: jest.fn().mockResolvedValue({ ready: true, state: 'ready' }), getEmbeddings: jest.fn(), }; const provider = await ProviderResolver.resolveWithValidation(); diff --git a/src/services/embeddings/ProviderResolver.ts b/src/services/embeddings/ProviderResolver.ts index 842b450..446d52b 100644 --- a/src/services/embeddings/ProviderResolver.ts +++ b/src/services/embeddings/ProviderResolver.ts @@ -1,21 +1,24 @@ import joplin from 'api'; import { EmbeddingProvider, ProviderConfig } from './Types'; -import { JoplinNativeProvider, JoplinAiApi } from './providers/JoplinNativeProvider'; +import { JoplinNativeProvider, JoplinAiApi, isIndexUsable } from './providers/JoplinNativeProvider'; export class ProviderResolver { /** * Resolves the native embedding provider after verifying that Joplin AI is - * available and its embedding index is ready. + * available and its embedding index is usable. */ public static async resolveWithValidation(): Promise { const joplinAi = joplin.ai as unknown as JoplinAiApi | undefined; - if (!joplinAi || typeof joplinAi.getIndexStatus !== 'function' || typeof joplinAi.getEmbeddings !== 'function') { + if (!joplinAi) { throw new Error('joplin.ai is not available. Enable AI in Settings → AI. Requires Joplin v3.7+.'); } const status = await joplinAi.getIndexStatus(); - if (!status || !status.ready) { - throw new Error('Joplin AI index is not ready. Enable AI and the embedding index in Settings → AI.'); + if (!status || !isIndexUsable(status.state)) { + throw new Error( + `Joplin AI index is not usable yet (state: ${status?.state ?? 'unknown'}). ` + + 'Enable AI and the embedding index in Settings → AI.' + ); } return new JoplinNativeProvider(status.modelId ?? JoplinNativeProvider.DEFAULT_MODEL_ID); } diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.test.ts b/src/services/embeddings/Providers/JoplinNativeProvider.test.ts index 63130a2..99a773b 100644 --- a/src/services/embeddings/Providers/JoplinNativeProvider.test.ts +++ b/src/services/embeddings/Providers/JoplinNativeProvider.test.ts @@ -9,7 +9,7 @@ describe('JoplinNativeProvider', () => { getEmbeddings: jest.Mock; }; - ai.getIndexStatus.mockResolvedValue({ ready: true, modelId: 'test-model' }); + ai.getIndexStatus.mockResolvedValue({ ready: true, state: 'ready', modelId: 'test-model' }); ai.getEmbeddings.mockResolvedValue({ modelId: 'test-model', dimension: 3, @@ -30,7 +30,7 @@ describe('JoplinNativeProvider', () => { getEmbeddings: jest.Mock; }; - ai.getIndexStatus.mockResolvedValue({ ready: true, modelId: 'fresh-model' }); + ai.getIndexStatus.mockResolvedValue({ ready: true, state: 'ready', modelId: 'fresh-model' }); ai.getEmbeddings.mockResolvedValue({ modelId: 'fresh-model', dimension: 2, @@ -56,7 +56,7 @@ describe('JoplinNativeProvider', () => { getEmbeddings: jest.Mock; }; - ai.getIndexStatus.mockResolvedValue({ ready: true, modelId: 'test-model' }); + ai.getIndexStatus.mockResolvedValue({ ready: true, state: 'ready', modelId: 'test-model' }); ai.getEmbeddings .mockResolvedValueOnce({ modelId: 'test-model', @@ -88,7 +88,7 @@ describe('JoplinNativeProvider', () => { getEmbeddings: jest.Mock; }; - ai.getIndexStatus.mockResolvedValue({ ready: true, modelId: 'model-a' }); + ai.getIndexStatus.mockResolvedValue({ ready: true, state: 'ready', modelId: 'model-a' }); ai.getEmbeddings .mockResolvedValueOnce({ modelId: 'model-a', @@ -117,4 +117,38 @@ describe('JoplinNativeProvider', () => { expect(provider.getFetchedModelId()).toBe('model-b'); expect(provider.modelName).toBe('model-b'); }); + + it('fetches vectors while the index is still indexing, since results are just partial', async () => { + const provider = new JoplinNativeProvider(); + const ai = joplin.ai as unknown as { + getIndexStatus: jest.Mock; + getEmbeddings: jest.Mock; + }; + + ai.getIndexStatus.mockResolvedValue({ ready: false, state: 'indexing', modelId: 'test-model' }); + ai.getEmbeddings.mockResolvedValue({ + modelId: 'test-model', + dimension: 2, + chunks: [{ noteId: 'n1', vector: [1, 0] }], + nextCursor: undefined, + }); + + const vectors = await provider.fetchVectorsByNoteIds(['n1']); + + expect(vectors.get('n1')).toEqual([1, 0]); + }); + + it('throws when the index is disabled', async () => { + const provider = new JoplinNativeProvider(); + const ai = joplin.ai as unknown as { + getIndexStatus: jest.Mock; + getEmbeddings: jest.Mock; + }; + + ai.getIndexStatus.mockResolvedValue({ ready: false, state: 'disabled', modelId: null }); + + await expect(provider.fetchVectorsByNoteIds(['n1'])).rejects.toThrow( + 'Joplin AI index is not usable yet (state: disabled)' + ); + }); }); diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.ts b/src/services/embeddings/Providers/JoplinNativeProvider.ts index ed6a47a..9a13758 100644 --- a/src/services/embeddings/Providers/JoplinNativeProvider.ts +++ b/src/services/embeddings/Providers/JoplinNativeProvider.ts @@ -1,18 +1,52 @@ import joplin from 'api'; import { EmbeddingProvider, ProviderId } from '../Types'; +/** + * Mirrors Joplin's official AiIndexState type (joplinapp.org/api/references/plugin_api). + * 'unavailable' | 'disabled' | 'preparing' block any fetch (no data yet). + * 'indexing' still allows fetching — results are partial, handled downstream + * as per-note "not yet indexed" errors. 'ready' is the fully-indexed state. + */ +export type AiIndexState = 'unavailable' | 'disabled' | 'preparing' | 'indexing' | 'ready'; + +export interface AiIndexStatus { + modelId: string | null; + notesIndexed: number; + ready: boolean; + state: AiIndexState; + totalNotes: number; +} + +export interface EmbeddingChunk { + chunkIndex: number; + chunkText: string; + noteId: string; + vector: number[]; +} + +export interface EmbeddingsPage { + chunks: EmbeddingChunk[]; + dimension: number; + modelId: string; + nextCursor?: string; +} + +export interface GetEmbeddingsOptions { + cursor?: string; + limit?: number; + noteIds?: string[]; +} + export interface JoplinAiApi { - getIndexStatus: () => Promise<{ ready: boolean; modelId?: string | null }>; - getEmbeddings: (params: { - noteIds?: string[]; - cursor?: string; - limit: number; - }) => Promise<{ - modelId?: string | null; - dimension: number; - chunks: Array<{ noteId: string; vector: number[] }>; - nextCursor?: string; - }>; + getIndexStatus: () => Promise; + getEmbeddings: (options: GetEmbeddingsOptions) => Promise; +} + +const BLOCKING_STATES: ReadonlySet = new Set(['unavailable', 'disabled', 'preparing']); + +/** True once the index has enough data to fetch from, even if still indexing. */ +export function isIndexUsable(state: AiIndexState | undefined): boolean { + return !!state && !BLOCKING_STATES.has(state); } export class JoplinNativeProvider implements EmbeddingProvider { @@ -60,15 +94,20 @@ export class JoplinNativeProvider implements EmbeddingProvider { return this.fetchedModelId; } - /** Checks that joplin.ai exists and has the required methods. */ + /** + * Checks that joplin.ai exists. Deliberately does not probe individual + * method properties (e.g. `typeof api.getEmbeddings`) — Joplin's plugin + * RPC bridge exposes joplin.ai as a proxy that accumulates property-path + * state across accesses, so a property read that's never invoked can + * corrupt the path used by a later real call. Always access a method and + * invoke it in the same expression; let a genuinely missing method throw + * on invocation instead of pre-checking with typeof. + */ private validateAiApi(): JoplinAiApi { const api = joplin.ai as unknown as JoplinAiApi | undefined; - if (!api || typeof api.getEmbeddings !== 'function') { - throw new Error('joplin.ai.getEmbeddings is not available. Enable AI in Settings → AI.'); - } - if (typeof api.getIndexStatus !== 'function') { - throw new Error('joplin.ai.getIndexStatus is not available. Enable AI in Settings → AI.'); + if (!api) { + throw new Error('joplin.ai is not available. Enable AI in Settings → AI.'); } return api; @@ -82,12 +121,7 @@ export class JoplinNativeProvider implements EmbeddingProvider { api: JoplinAiApi, noteIds: string[], ): Promise> { - const status = await api.getIndexStatus(); - if (!status || !status.ready) { - throw new Error('Joplin AI index is not ready. Wait for indexing to complete or enable AI in Settings → AI.'); - } - - let trackedModelId: string | null = status.modelId ?? null; + let trackedModelId = await this.requireUsableIndex(api); const grouped = new Map(); let cursor: string | undefined; @@ -123,14 +157,7 @@ export class JoplinNativeProvider implements EmbeddingProvider { } } - for (const chunk of page.chunks) { - const list = grouped.get(chunk.noteId); - if (list) { - list.push(chunk.vector); - } else { - grouped.set(chunk.noteId, [chunk.vector]); - } - } + this.addChunksToGroup(grouped, page.chunks); cursor = page.nextCursor; if (!cursor) { @@ -142,6 +169,30 @@ export class JoplinNativeProvider implements EmbeddingProvider { return grouped; } + /** Throws if the index isn't usable yet; otherwise returns the model ID it's currently indexed with. */ + private async requireUsableIndex(api: JoplinAiApi): Promise { + const status = await api.getIndexStatus(); + if (!status || !isIndexUsable(status.state)) { + throw new Error( + `Joplin AI index is not usable yet (state: ${status?.state ?? 'unknown'}). ` + + 'Enable AI and wait for the embedding model to finish loading in Settings → AI.' + ); + } + return status.modelId ?? null; + } + + /** Appends each chunk's vector onto its note's running vector list. */ + private addChunksToGroup(grouped: Map, chunks: EmbeddingChunk[]): void { + for (const chunk of chunks) { + const list = grouped.get(chunk.noteId); + if (list) { + list.push(chunk.vector); + } else { + grouped.set(chunk.noteId, [chunk.vector]); + } + } + } + /** Averages multiple chunk vectors per note into one vector and L2-normalizes. */ private poolAndNormalize(grouped: Map): Map { const result = new Map(); diff --git a/src/services/graph/GraphBuilder.test.ts b/src/services/graph/GraphBuilder.test.ts index 5c31c71..0864f56 100644 --- a/src/services/graph/GraphBuilder.test.ts +++ b/src/services/graph/GraphBuilder.test.ts @@ -1,10 +1,13 @@ import { GraphBuilder } from './GraphBuilder'; import { EdgeFactory } from '../similarity/EdgeFactory'; +import { SimilarityEngine } from '../similarity/SimilarityEngine'; import { Note } from '../../data/Types'; jest.mock('../similarity/EdgeFactory'); +jest.mock('../similarity/SimilarityEngine'); const MockEdgeFactory = EdgeFactory as jest.MockedClass; +const MockSimilarityEngine = SimilarityEngine as jest.MockedClass; function note( id: string, @@ -77,4 +80,45 @@ describe('GraphBuilder', () => { expect(result.edges).toHaveLength(1); expect(result.edges[0].data).toEqual({ source: 'a', target: 'b', type: 'link' }); }); + + describe('buildWithSimilarity', () => { + it('adds semantic edges computed from embeddings alongside structural edges', async () => { + mockEdgeFactory.createEdges.mockReturnValue([{ source: 'a', target: 'c', type: 'link' }]); + mockEdgeFactory.createSemanticEdges.mockReturnValue([ + { source: 'a', target: 'b', type: 'semantic' }, + ]); + MockSimilarityEngine.mockImplementation(() => ({ + compute: jest.fn().mockResolvedValue([{ source: 'a', target: 'b', score: 0.8 }]), + }) as unknown as SimilarityEngine); + + const notes = [note('a', 'A'), note('b', 'B'), note('c', 'C')]; + const embeddedNotes = [ + { note: notes[0], embedding: [1, 0] }, + { note: notes[1], embedding: [0.9, 0.1] }, + ]; + + const result = await builder.buildWithSimilarity(notes, embeddedNotes); + + expect(mockEdgeFactory.createSemanticEdges).toHaveBeenCalledWith([ + { source: 'a', target: 'b', score: 0.8 }, + ]); + expect(result.edges).toContainEqual({ data: { source: 'a', target: 'b', type: 'semantic' } }); + expect(result.edges).toContainEqual({ data: { source: 'a', target: 'c', type: 'link' } }); + expect(result.edges).toHaveLength(2); + }); + + it('still returns a graph when there are no semantic matches', async () => { + mockEdgeFactory.createEdges.mockReturnValue([]); + mockEdgeFactory.createSemanticEdges.mockReturnValue([]); + MockSimilarityEngine.mockImplementation(() => ({ + compute: jest.fn().mockResolvedValue([]), + }) as unknown as SimilarityEngine); + + const notes = [note('a', 'A'), note('b', 'B')]; + const result = await builder.buildWithSimilarity(notes, []); + + expect(result.nodes).toHaveLength(2); + expect(result.edges).toEqual([]); + }); + }); }); diff --git a/src/services/graph/GraphBuilder.ts b/src/services/graph/GraphBuilder.ts index 1921cfa..88f1526 100644 --- a/src/services/graph/GraphBuilder.ts +++ b/src/services/graph/GraphBuilder.ts @@ -1,6 +1,8 @@ import { Note } from '../../data/Types'; import { EdgeFactory } from '../similarity/EdgeFactory'; -import { GraphData, GraphNode } from './types'; +import { SimilarityEngine } from '../similarity/SimilarityEngine'; +import { EmbeddedNote } from '../embeddings/Types'; +import { GraphData, GraphEdge, GraphNode } from './types'; export class GraphBuilder { private readonly edgeFactory: EdgeFactory; @@ -15,20 +17,57 @@ export class GraphBuilder { * @returns graph data ready for rendering (nodes and edges). */ public build(notes: Note[]): GraphData { + const edges = this.edgeFactory.createEdges(notes); + return this.buildData(notes, edges); + } + + /** + * Builds a graph with semantic edges computed from embedding vectors, + * in addition to link and tag edges. + */ + public async buildWithSimilarity( + notes: Note[], + embeddedNotes: EmbeddedNote[], + ): Promise { + const structuralEdges = this.edgeFactory.createEdges(notes); + + const engine = new SimilarityEngine(notes, embeddedNotes); + const pairs = await engine.compute(); + const semanticEdges = this.edgeFactory.createSemanticEdges(pairs); + + const allEdges = [...structuralEdges, ...semanticEdges]; + return this.buildData(notes, allEdges); + } + + private buildData(notes: Note[], edges: GraphEdge[]): GraphData { + const degreeMap = this.computeDegreeMap(notes, edges); + const nodes = this.buildNodes(notes, degreeMap); + + const nodeIdSet = new Set(nodes.map((n) => n.data.id)); + const visibleEdges = this.filterVisibleEdges(edges, nodeIdSet); + + this.logGraphStats(nodes, visibleEdges, degreeMap); + + return { nodes, edges: visibleEdges.map((e) => ({ data: e })) }; + } + + /** Counts each note's connections, including notes an edge references that aren't in `notes`. */ + private computeDegreeMap(notes: Note[], edges: GraphEdge[]): Map { const degreeMap = new Map(); for (const note of notes) { degreeMap.set(note.id, 0); } - const edges = this.edgeFactory.createEdges(notes); - for (const edge of edges) { degreeMap.set(edge.source, (degreeMap.get(edge.source) ?? 0) + 1); degreeMap.set(edge.target, (degreeMap.get(edge.target) ?? 0) + 1); } - const maxDegree = Math.max(1, ...degreeMap.values()); + return degreeMap; + } + /** Builds one node per note, truncating long titles to keep labels readable in the graph. */ + private buildNodes(notes: Note[], degreeMap: Map): Array<{ data: GraphNode }> { const nodes: Array<{ data: GraphNode }> = []; for (const note of notes) { const degree = degreeMap.get(note.id) ?? 0; @@ -42,24 +81,30 @@ export class GraphBuilder { }, }); } + return nodes; + } - const nodeIdSet = new Set(nodes.map((n) => n.data.id)); - const visibleEdges = edges.filter( - (e) => nodeIdSet.has(e.source) && nodeIdSet.has(e.target) - ); + /** Drops edges referencing a note outside the current node set. */ + private filterVisibleEdges(edges: GraphEdge[], nodeIdSet: Set): GraphEdge[] { + return edges.filter((e) => nodeIdSet.has(e.source) && nodeIdSet.has(e.target)); + } + private logGraphStats( + nodes: Array<{ data: GraphNode }>, + visibleEdges: GraphEdge[], + degreeMap: Map, + ): void { const connectedIds = new Set(); for (const edge of visibleEdges) { connectedIds.add(edge.source); connectedIds.add(edge.target); } const isolatedCount = nodes.length - connectedIds.size; + const maxDegree = Math.max(1, ...degreeMap.values()); console.info( `Graph built: ${nodes.length} nodes, ${visibleEdges.length} edges ` + `(${isolatedCount} isolated, max degree ${maxDegree})` ); - - return { nodes, edges: visibleEdges.map((e) => ({ data: e })) }; } } diff --git a/src/services/similarity/EdgeFactory.test.ts b/src/services/similarity/EdgeFactory.test.ts index 90c1924..c753663 100644 --- a/src/services/similarity/EdgeFactory.test.ts +++ b/src/services/similarity/EdgeFactory.test.ts @@ -101,4 +101,25 @@ describe('EdgeFactory', () => { it('ignores self-referencing links', () => { expect(factory.createEdges([note('a', 'A', ['a'])])).toEqual([]); }); + + describe('createSemanticEdges', () => { + it('returns empty for no pairs', () => { + expect(factory.createSemanticEdges([])).toEqual([]); + }); + + it('creates a semantic edge for each positive-score pair', () => { + const edges = factory.createSemanticEdges([ + { source: 'a', target: 'b', score: 0.8 }, + ]); + expect(edges).toEqual([{ source: 'a', target: 'b', type: 'semantic' }]); + }); + + it('excludes pairs with a non-positive score', () => { + const edges = factory.createSemanticEdges([ + { source: 'a', target: 'b', score: 0 }, + { source: 'c', target: 'd', score: -0.1 }, + ]); + expect(edges).toEqual([]); + }); + }); }); diff --git a/src/services/similarity/EdgeFactory.ts b/src/services/similarity/EdgeFactory.ts index 585c344..0197272 100644 --- a/src/services/similarity/EdgeFactory.ts +++ b/src/services/similarity/EdgeFactory.ts @@ -1,5 +1,6 @@ import { Note } from '../../data/Types'; import { GraphEdge } from '../graph/types'; +import { SimilarityPair } from './SimilarityEngine'; export class EdgeFactory { /** @@ -8,6 +9,11 @@ export class EdgeFactory { * @returns deduplicated edges of type `link` and `tag`. */ public createEdges(notes: Note[]): GraphEdge[] { + return [...this.createLinkEdges(notes), ...this.createTagEdges(notes)]; + } + + /** Builds one deduplicated edge per explicit `:/noteId` link between two notes in scope. */ + private createLinkEdges(notes: Note[]): GraphEdge[] { const noteIdSet = new Set(notes.map((n) => n.id)); const edges: GraphEdge[] = []; const linkKeySet = new Set(); @@ -24,17 +30,18 @@ export class EdgeFactory { } } - const tagToNotes = new Map(); - for (const note of notes) { - for (const tag of note.tags ?? []) { - if (!tagToNotes.has(tag)) { - tagToNotes.set(tag, []); - } - tagToNotes.get(tag)!.push(note.id); - } - } + return edges; + } + /** + * Builds one edge per pair of notes sharing a tag, merging multiple shared + * tag names onto the same edge. Tags shared by more than 20 notes are + * skipped to avoid a combinatorial blowup of pairs. + */ + private createTagEdges(notes: Note[]): GraphEdge[] { + const tagToNotes = this.groupNoteIdsByTag(notes); const tagEdgeMap = new Map(); + for (const [tagName, noteIds] of tagToNotes) { if (noteIds.length > 20) continue; @@ -59,8 +66,37 @@ export class EdgeFactory { } } - for (const edge of tagEdgeMap.values()) { - edges.push(edge); + return Array.from(tagEdgeMap.values()); + } + + private groupNoteIdsByTag(notes: Note[]): Map { + const tagToNotes = new Map(); + for (const note of notes) { + for (const tag of note.tags ?? []) { + if (!tagToNotes.has(tag)) { + tagToNotes.set(tag, []); + } + tagToNotes.get(tag)!.push(note.id); + } + } + return tagToNotes; + } + + /** + * Creates semantic edges from similarity pairs. + * Each pair represents a strong semantic connection between two notes. + */ + public createSemanticEdges(pairs: SimilarityPair[]): GraphEdge[] { + const edges: GraphEdge[] = []; + + for (const pair of pairs) { + if (pair.score <= 0) continue; + + edges.push({ + source: pair.source, + target: pair.target, + type: 'semantic', + }); } return edges; diff --git a/src/services/similarity/SimilarityEngine.test.ts b/src/services/similarity/SimilarityEngine.test.ts new file mode 100644 index 0000000..f9c8f1c --- /dev/null +++ b/src/services/similarity/SimilarityEngine.test.ts @@ -0,0 +1,515 @@ +import { SimilarityEngine } from './SimilarityEngine'; +import { Note } from '../../data/Types'; +import { EmbeddedNote } from '../embeddings/Types'; + +function makeNote( + id: string, + title: string, + links: string[] = [], + tags: string[] = [], + createdTime = 0, +): Note { + return { + id, + parent_id: 'p1', + title, + body: '', + created_time: createdTime, + updated_time: 1, + links, + tags, + }; +} + +const DAY_MS = 1000 * 60 * 60 * 24; + +function embed(id: string, vector: number[]): EmbeddedNote { + return { note: makeNote(id, 'Note ' + id), embedding: vector }; +} + +describe('SimilarityEngine', () => { + describe('compute', () => { + it('returns empty for no notes', async () => { + const engine = new SimilarityEngine([], []); + const pairs = await engine.compute(); + expect(pairs).toEqual([]); + }); + + it('returns empty for a single note', async () => { + const engine = new SimilarityEngine( + [makeNote('a', 'A')], + [embed('a', [1, 0, 0])], + ); + const pairs = await engine.compute(); + expect(pairs).toEqual([]); + }); + + it('computes cosine similarity for two similar notes', async () => { + const notes = [makeNote('a', 'A'), makeNote('b', 'B')]; + const embedded = [embed('a', [1, 0]), embed('b', [0.95, 0.3])]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + expect(pairs).toHaveLength(1); + expect(pairs[0].source).toBe('a'); + expect(pairs[0].target).toBe('b'); + expect(pairs[0].score).toBeGreaterThan(0.5); + }); + + it('orders source before target deterministically', async () => { + const notes = [makeNote('b', 'B'), makeNote('a', 'A')]; + const embedded = [embed('b', [1, 0]), embed('a', [0.95, 0.3])]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + expect(pairs).toHaveLength(1); + expect(pairs[0].source).toBe('a'); + expect(pairs[0].target).toBe('b'); + }); + + it('returns empty when all notes lack vectors', async () => { + const notes = [makeNote('a', 'A'), makeNote('b', 'B')]; + const engine = new SimilarityEngine(notes, []); + const pairs = await engine.compute(); + expect(pairs).toEqual([]); + }); + + it('skips notes without vectors in the mapping', async () => { + const notes = [makeNote('a', 'A'), makeNote('b', 'B'), makeNote('c', 'C')]; + const embedded = [embed('a', [0.95, 0.3]), embed('c', [0.3, 0.95])]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const involvesB = pairs.some( + p => p.source === 'b' || p.target === 'b', + ); + expect(involvesB).toBe(false); + }); + }); + + describe('normalization', () => { + it('produces scores in [0, 1] range', async () => { + const notes = [makeNote('a', 'A'), makeNote('b', 'B'), makeNote('c', 'C')]; + const embedded = [ + embed('a', [1, 0]), + embed('b', [0.95, 0.3]), + embed('c', [0.3, 0.95]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + for (const p of pairs) { + expect(p.score).toBeGreaterThanOrEqual(0); + expect(p.score).toBeLessThanOrEqual(1.5); + } + }); + + it('gives higher scores to more similar notes', async () => { + const notes = [ + makeNote('a', 'A'), makeNote('b', 'B'), + makeNote('c', 'C'), makeNote('d', 'D'), + ]; + const embedded = [ + embed('a', [1, 0]), + embed('b', [0.95, 0.31]), + embed('c', [0.7, 0.71]), + embed('d', [-1, 0]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const abScore = pairs.find( + p => + (p.source === 'a' && p.target === 'b') || + (p.source === 'b' && p.target === 'a'), + ); + + const acScore = pairs.find( + p => + (p.source === 'a' && p.target === 'c') || + (p.source === 'c' && p.target === 'a'), + ); + + expect(abScore).toBeDefined(); + expect(acScore).toBeDefined(); + expect(abScore!.score).toBeGreaterThan(acScore!.score); + }); + }); + + describe('tag bonuses', () => { + it('boosts score when notes share tags', async () => { + // Padded to 7 notes so the 'shared' tag (on 2 of them) stays under the + // 30% organizational-tag threshold and isn't excluded from the signal. + const notes = [ + makeNote('a', 'A', [], ['shared']), + makeNote('b', 'B', [], ['shared']), + makeNote('c', 'C', [], []), + makeNote('pad0', 'Pad 0'), + makeNote('pad1', 'Pad 1'), + makeNote('pad2', 'Pad 2'), + makeNote('pad3', 'Pad 3'), + ]; + const embedded = [ + embed('a', [0.95, 0.3]), + embed('b', [0.98, 0.2]), + embed('c', [0.96, 0.28]), + embed('pad0', [-1, 0]), + embed('pad1', [0, -1]), + embed('pad2', [-0.7, 0.7]), + embed('pad3', [0.7, -0.7]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const abWithTag = pairs.find( + p => + (p.source === 'a' && p.target === 'b') || + (p.source === 'b' && p.target === 'a'), + ); + const acNoTag = pairs.find( + p => + (p.source === 'a' && p.target === 'c') || + (p.source === 'c' && p.target === 'a'), + ); + + expect(abWithTag).toBeDefined(); + expect(acNoTag).toBeDefined(); + expect(abWithTag!.score).toBeGreaterThan(acNoTag!.score); + }); + + it('scales the tag bonus by Jaccard overlap, not raw shared-tag count', async () => { + // a-b and c-d each have the highest raw dot product (0.9) in the whole + // fixture, so both normalize to exactly 1.0 regardless of the padding + // notes' spread — isolating the tag bonus as the only source of + // difference between their final scores. a/b fully overlap in tags + // (jaccard=1.0 -> bonus=0.1); c/d partially overlap (jaccard=0.5 -> + // bonus=0.05). A raw-count bonus would instead give both pairs the + // same 2 * TAG_BONUS = 0.2, making them equal. + const notes = [ + makeNote('a', 'A', [], ['p', 'q']), + makeNote('b', 'B', [], ['p', 'q']), + makeNote('c', 'C', [], ['r', 's']), + makeNote('d', 'D', [], ['r', 's', 't', 'u']), + makeNote('pad0', 'Pad 0'), + makeNote('pad1', 'Pad 1'), + makeNote('pad2', 'Pad 2'), + makeNote('pad3', 'Pad 3'), + ]; + const embedded = [ + embed('a', [1, 0, 0, 0]), + embed('b', [0.9, Math.sqrt(1 - 0.81), 0, 0]), + embed('c', [0, 1, 0, 0]), + embed('d', [0, 0.9, Math.sqrt(1 - 0.81), 0]), + embed('pad0', [-1, 0, 0, 0]), + embed('pad1', [0, -1, 0, 0]), + embed('pad2', [0, 0, -1, 0]), + embed('pad3', [0, 0, 0, 1]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const ab = pairs.find( + p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + ); + const cd = pairs.find( + p => (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c'), + ); + + expect(ab).toBeDefined(); + expect(cd).toBeDefined(); + expect(ab!.score - cd!.score).toBeCloseTo(0.05, 5); + }); + + it('excludes organizational tags (present on more than 30% of notes) from the bonus', async () => { + // 'inbox' appears on a, c, d, e (4 of 10 notes = 40%) — organizational, excluded. + // 'project' appears only on a and b (2 of 10 = 20%) — meaningful, included. + const notes = [ + makeNote('a', 'A', [], ['inbox', 'project']), + makeNote('b', 'B', [], ['project']), + makeNote('c', 'C', [], ['inbox']), + makeNote('d', 'D', [], ['inbox']), + makeNote('e', 'E', [], ['inbox']), + makeNote('pad0', 'Pad 0'), + makeNote('pad1', 'Pad 1'), + makeNote('pad2', 'Pad 2'), + makeNote('pad3', 'Pad 3'), + makeNote('pad4', 'Pad 4'), + ]; + const embedded = [ + embed('a', [0.95, 0.3]), + embed('b', [0.98, 0.2]), + embed('c', [0.96, 0.28]), + embed('d', [-1, 0]), + embed('e', [0, -1]), + embed('pad0', [-0.7, 0.7]), + embed('pad1', [0.7, -0.7]), + embed('pad2', [-0.9, 0.1]), + embed('pad3', [0.1, -0.9]), + embed('pad4', [-0.5, -0.5]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const abSharesProject = pairs.find( + p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + ); + const acSharesOnlyInbox = pairs.find( + p => (p.source === 'a' && p.target === 'c') || (p.source === 'c' && p.target === 'a'), + ); + + expect(abSharesProject).toBeDefined(); + expect(acSharesOnlyInbox).toBeDefined(); + expect(abSharesProject!.score).toBeGreaterThan(acSharesOnlyInbox!.score); + }); + }); + + describe('link bonuses', () => { + it('boosts score when notes link to each other', async () => { + const notes = [ + makeNote('a', 'A', ['b'], []), + makeNote('b', 'B', [], []), + makeNote('c', 'C', [], []), + ]; + const embedded = [ + embed('a', [0.95, 0.3]), + embed('b', [0.98, 0.2]), + embed('c', [0.96, 0.28]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const abWithLink = pairs.find( + p => + (p.source === 'a' && p.target === 'b') || + (p.source === 'b' && p.target === 'a'), + ); + const acNoLink = pairs.find( + p => + (p.source === 'a' && p.target === 'c') || + (p.source === 'c' && p.target === 'a'), + ); + + expect(abWithLink).toBeDefined(); + expect(acNoLink).toBeDefined(); + expect(abWithLink!.score).toBeGreaterThan(acNoLink!.score); + }); + }); + + describe('temporal proximity bonus', () => { + it('gives a stronger boost to notes created within a day than notes created within a week', async () => { + // a-b and c-d each have the same raw dot product (0.9) in the whole + // fixture, so both normalize to exactly 1.0 regardless of the padding + // notes' spread — isolating the temporal bonus as the only source of + // difference between their final scores. a/b were created 12 hours + // apart (same-day bonus 0.1); c/d were created 3 days apart (same-week + // bonus 0.05). + const notes = [ + makeNote('a', 'A', [], [], 0), + makeNote('b', 'B', [], [], 12 * 60 * 60 * 1000), + makeNote('c', 'C', [], [], 0), + makeNote('d', 'D', [], [], 3 * DAY_MS), + makeNote('pad0', 'Pad 0'), + makeNote('pad1', 'Pad 1'), + makeNote('pad2', 'Pad 2'), + makeNote('pad3', 'Pad 3'), + ]; + const embedded = [ + embed('a', [1, 0, 0, 0]), + embed('b', [0.9, Math.sqrt(1 - 0.81), 0, 0]), + embed('c', [0, 1, 0, 0]), + embed('d', [0, 0.9, Math.sqrt(1 - 0.81), 0]), + embed('pad0', [-1, 0, 0, 0]), + embed('pad1', [0, -1, 0, 0]), + embed('pad2', [0, 0, -1, 0]), + embed('pad3', [0, 0, 0, 1]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const ab = pairs.find( + p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + ); + const cd = pairs.find( + p => (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c'), + ); + + expect(ab).toBeDefined(); + expect(cd).toBeDefined(); + expect(ab!.score - cd!.score).toBeCloseTo(0.05, 5); + }); + + it('gives no temporal bonus to notes created more than a week apart', async () => { + const notes = [ + makeNote('a', 'A', [], [], 0), + makeNote('b', 'B', [], [], DAY_MS), + makeNote('c', 'C', [], [], 0), + makeNote('d', 'D', [], [], 30 * DAY_MS), + makeNote('pad0', 'Pad 0'), + makeNote('pad1', 'Pad 1'), + makeNote('pad2', 'Pad 2'), + makeNote('pad3', 'Pad 3'), + ]; + const embedded = [ + embed('a', [1, 0, 0, 0]), + embed('b', [0.9, Math.sqrt(1 - 0.81), 0, 0]), + embed('c', [0, 1, 0, 0]), + embed('d', [0, 0.9, Math.sqrt(1 - 0.81), 0]), + embed('pad0', [-1, 0, 0, 0]), + embed('pad1', [0, -1, 0, 0]), + embed('pad2', [0, 0, -1, 0]), + embed('pad3', [0, 0, 0, 1]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const ab = pairs.find( + p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + ); + const cd = pairs.find( + p => (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c'), + ); + + expect(ab).toBeDefined(); + expect(cd).toBeDefined(); + expect(ab!.score - cd!.score).toBeCloseTo(0.1, 5); + }); + + it('applies the bonus inclusively at exactly 1 day and exactly 7 days', async () => { + const notes = [ + makeNote('a', 'A', [], [], 0), + makeNote('b', 'B', [], [], DAY_MS), + makeNote('c', 'C', [], [], 0), + makeNote('d', 'D', [], [], 7 * DAY_MS), + makeNote('pad0', 'Pad 0'), + makeNote('pad1', 'Pad 1'), + makeNote('pad2', 'Pad 2'), + makeNote('pad3', 'Pad 3'), + ]; + const embedded = [ + embed('a', [1, 0, 0, 0]), + embed('b', [0.9, Math.sqrt(1 - 0.81), 0, 0]), + embed('c', [0, 1, 0, 0]), + embed('d', [0, 0.9, Math.sqrt(1 - 0.81), 0]), + embed('pad0', [-1, 0, 0, 0]), + embed('pad1', [0, -1, 0, 0]), + embed('pad2', [0, 0, -1, 0]), + embed('pad3', [0, 0, 0, 1]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const ab = pairs.find( + p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + ); + const cd = pairs.find( + p => (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c'), + ); + + expect(ab).toBeDefined(); + expect(cd).toBeDefined(); + expect(ab!.score - cd!.score).toBeCloseTo(0.05, 5); + }); + }); + + describe('top-K filtering', () => { + it('limits edges per note', async () => { + const notes = []; + const embedded = []; + for (let i = 0; i < 6; i++) { + notes.push(makeNote(`n${i}`, `Note ${i}`)); + embedded.push( + embed(`n${i}`, [ + Math.cos((i * Math.PI) / 3), + Math.sin((i * Math.PI) / 3), + ]), + ); + } + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + for (const note of notes) { + const edges = pairs.filter( + p => p.source === note.id || p.target === note.id, + ); + expect(edges.length).toBeLessThanOrEqual(5); + } + }); + + it('lets a hub note exceed K edges when more than K notes independently pick it', async () => { + // A shares a "topic" dimension with 8 satellites, and each satellite + // also has its own unique dimension. That makes every satellite closer + // to A (dot = 0.9) than to any other satellite (dot = 0.81), so A is + // always each satellite's #1 pick. Top-K is per-note (union), not a + // hard cap on incoming edges, so A ends up with more than 5 edges here. + const dims = 9; + const aVector = new Array(dims).fill(0); + aVector[0] = 1; + + const notes = [makeNote('a', 'A')]; + const embedded = [embed('a', aVector)]; + + for (let i = 0; i < 8; i++) { + const satelliteVector = new Array(dims).fill(0); + satelliteVector[0] = 0.9; + satelliteVector[i + 1] = 0.3; + notes.push(makeNote(`n${i}`, `Note ${i}`)); + embedded.push(embed(`n${i}`, satelliteVector)); + } + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + const aEdges = pairs.filter(p => p.source === 'a' || p.target === 'a'); + expect(aEdges.length).toBeGreaterThan(5); + }); + }); + + describe('SEMANTIC_FLOOR and threshold ordering', () => { + it('rejects a below-floor pair even with a shared tag', async () => { + // a and b are nearly orthogonal (cosine ~0), share a tag but are not + // linked. SEMANTIC_FLOOR must reject them before bonuses or threshold + // ever apply — tags alone can never manufacture an edge out of a weak + // semantic score. + const notes = [ + makeNote('a', 'A', [], ['shared']), + makeNote('b', 'B', [], ['shared']), + ]; + const embedded = [embed('a', [1, 0]), embed('b', [0, 1])]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + expect(pairs).toEqual([]); + }); + + it('lets a direct link bypass the floor, but the boosted score must still clear the threshold', async () => { + // a and b are nearly orthogonal (cosine ~0) but directly link to each other + // and share a tag. The link bypasses SEMANTIC_FLOOR (a user-created edge + // isn't a false positive), but the resulting boosted score (~0.25) still + // isn't enough to clear DEFAULT_THRESHOLD (0.5). + const notes = [ + makeNote('a', 'A', ['b'], ['shared']), + makeNote('b', 'B', [], ['shared']), + ]; + const embedded = [embed('a', [1, 0]), embed('b', [0, 1])]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + expect(pairs).toEqual([]); + }); + }); +}); diff --git a/src/services/similarity/SimilarityEngine.ts b/src/services/similarity/SimilarityEngine.ts new file mode 100644 index 0000000..dd8a839 --- /dev/null +++ b/src/services/similarity/SimilarityEngine.ts @@ -0,0 +1,316 @@ +import joplin from 'api'; +import { SearchOptions, SearchResult } from 'api/types'; +import { Note } from '../../data/Types'; +import { EmbeddedNote } from '../embeddings/Types'; +import { + SEMANTIC_FLOOR, + DEFAULT_THRESHOLD, + TOP_K, + LARGE_VAULT_THRESHOLD, + TAG_BONUS, + LINK_BONUS, + ORGANIZATIONAL_TAG_RATIO, + TEMPORAL_BONUS_1_DAY, + TEMPORAL_BONUS_7_DAYS, + MS_PER_DAY, +} from './ThresholdPresets'; + +export interface SimilarityPair { + source: string; + target: string; + score: number; +} + +export class SimilarityEngine { + private readonly noteIds: string[]; + private readonly vectors: Map; + private readonly tagMap: Map>; + private readonly linkSet: Set; + private readonly createdTimeMap: Map; + + public constructor(notes: Note[], embeddedNotes: EmbeddedNote[]) { + this.noteIds = notes.map(n => n.id); + this.vectors = new Map(); + for (const en of embeddedNotes) { + this.vectors.set(en.note.id, en.embedding); + } + this.tagMap = this.buildTagMap(notes); + this.linkSet = this.buildLinkSet(notes); + this.createdTimeMap = new Map(notes.map(n => [n.id, n.created_time])); + } + + /** Orchestrates the full similarity pipeline: compute → normalize → floor → enrich → threshold → top-K. */ + public async compute(): Promise { + if (this.noteIds.length <= 1) { + return []; + } + + const rawPairs = await this.computeRawPairs(); + + if (rawPairs.length === 0) { + return []; + } + + const normalized = this.normalize(rawPairs); + const aboveFloor = this.filterBelowFloor(normalized, SEMANTIC_FLOOR); + const enriched = this.addBonusPoints(aboveFloor); + const aboveThreshold = this.filterBelowThreshold(enriched, DEFAULT_THRESHOLD); + const topPairs = this.selectTopK(aboveThreshold, TOP_K); + + return topPairs; + } + + /** Picks the appropriate similarity strategy based on vault size. */ + private computeRawPairs(): Promise { + if (this.noteIds.length <= LARGE_VAULT_THRESHOLD) { + return Promise.resolve(this.computeCosinePairs()); + } + return this.computeSearchPairs(); + } + + /** O(n²) pairwise cosine similarity via dot product on unit-norm vectors. */ + private computeCosinePairs(): SimilarityPair[] { + const pairs: SimilarityPair[] = []; + const n = this.noteIds.length; + + for (let i = 0; i < n; i++) { + const a = this.noteIds[i]; + const vecA = this.vectors.get(a); + if (!vecA) continue; + + for (let j = i + 1; j < n; j++) { + const b = this.noteIds[j]; + const vecB = this.vectors.get(b); + if (!vecB) continue; + + const score = this.dotProduct(vecA, vecB); + const [source, target] = a < b ? [a, b] : [b, a]; + pairs.push({ source, target, score }); + } + } + + return pairs; + } + + /** + * Uses joplin.ai.search({ noteId }) to find candidate pairs via vector index. + * Only checks that joplin.ai itself exists — never probes a specific method + * property without invoking it (see JoplinNativeProvider.validateAiApi for why). + */ + private async computeSearchPairs(): Promise { + const joplinAi = joplin.ai as unknown as + | { search: (options: SearchOptions) => Promise } + | undefined; + if (!joplinAi) { + return this.computeCosinePairs(); + } + + const seen = new Set(); + const pairs: SimilarityPair[] = []; + + for (const noteId of this.noteIds) { + try { + const results = await joplinAi.search({ + query: { noteId }, + relevance: 'normal', + }); + + for (const r of results) { + if (!this.vectors.has(r.noteId) || r.noteId === noteId) { + continue; + } + + const key = this.makePairKey(noteId, r.noteId); + if (seen.has(key)) continue; + seen.add(key); + + const [source, target] = noteId < r.noteId + ? [noteId, r.noteId] + : [r.noteId, noteId]; + + pairs.push({ source, target, score: r.score }); + } + } catch { + continue; + } + } + + return pairs; + } + + /** Dot product of two same-length vectors. */ + private dotProduct(a: number[], b: number[]): number { + let sum = 0; + for (let i = 0; i < a.length; i++) { + sum += a[i] * b[i]; + } + return sum; + } + + /** Min-max normalizes scores to [0, 1]. Skips if spread is too narrow. */ + private normalize(pairs: SimilarityPair[]): SimilarityPair[] { + let min = Infinity; + let max = -Infinity; + + for (const p of pairs) { + if (p.score < min) min = p.score; + if (p.score > max) max = p.score; + } + + const spread = max - min; + if (spread < 0.1) { + return pairs; + } + + for (const p of pairs) { + p.score = (p.score - min) / spread; + } + + return pairs; + } + + /** Adds shared-tag, direct-link, and temporal-proximity bonuses to each pair's score. */ + private addBonusPoints(pairs: SimilarityPair[]): SimilarityPair[] { + for (const p of pairs) { + p.score += this.sharedTagBonus(p) + this.directLinkBonus(p) + this.temporalProximityBonus(p); + } + return pairs; + } + + /** Jaccard overlap (intersection / union) between two notes' tag sets, scaled by TAG_BONUS. */ + private sharedTagBonus(pair: SimilarityPair): number { + const tagsA = this.tagMap.get(pair.source) ?? new Set(); + const tagsB = this.tagMap.get(pair.target) ?? new Set(); + if (tagsA.size === 0 && tagsB.size === 0) return 0; + + let intersectionSize = 0; + for (const t of tagsA) { + if (tagsB.has(t)) intersectionSize++; + } + const unionSize = new Set([...tagsA, ...tagsB]).size; + + return (intersectionSize / unionSize) * TAG_BONUS; + } + + private directLinkBonus(pair: SimilarityPair): number { + return this.isDirectlyLinked(pair) ? LINK_BONUS : 0; + } + + private isDirectlyLinked(pair: SimilarityPair): boolean { + return this.linkSet.has(this.makePairKey(pair.source, pair.target)); + } + + /** Boosts notes created close together in time: same day scores higher than same week. */ + private temporalProximityBonus(pair: SimilarityPair): number { + const createdA = this.createdTimeMap.get(pair.source); + const createdB = this.createdTimeMap.get(pair.target); + if (createdA === undefined || createdB === undefined) return 0; + + const daysApart = Math.abs(createdA - createdB) / MS_PER_DAY; + if (daysApart <= 1) return TEMPORAL_BONUS_1_DAY; + if (daysApart <= 7) return TEMPORAL_BONUS_7_DAYS; + return 0; + } + + /** + * Removes pairs below the safety floor — unless the notes are directly + * linked, in which case they're kept and left for the threshold check + * below. SEMANTIC_FLOOR guards against spurious tag-only edges, not + * against edges the user already created explicitly. + */ + private filterBelowFloor(pairs: SimilarityPair[], floor: number): SimilarityPair[] { + return pairs.filter(p => p.score >= floor || this.isDirectlyLinked(p)); + } + + /** Keeps only pairs whose bonus-boosted score clears the threshold. */ + private filterBelowThreshold(pairs: SimilarityPair[], threshold: number): SimilarityPair[] { + return pairs.filter(p => p.score >= threshold); + } + + /** + * Keeps each note's own K strongest connections; the returned edge set is + * their union, so a note that many others pick as one of their top-K can + * end up with more than K edges. This is the standard k-nearest-neighbor + * graph definition and preserves degree as a centrality signal. + */ + private selectTopK(pairs: SimilarityPair[], k: number): SimilarityPair[] { + const bySource = new Map(); + + for (const p of pairs) { + this.appendPair(bySource, p.source, p); + this.appendPair(bySource, p.target, { source: p.target, target: p.source, score: p.score }); + } + + const deduped = new Map(); + + for (const [, candidates] of bySource) { + candidates.sort((a, b) => b.score - a.score); + const kept = candidates.slice(0, k); + + for (const p of kept) { + const key = this.makePairKey(p.source, p.target); + if (!deduped.has(key)) { + deduped.set(key, p); + } + } + } + + return Array.from(deduped.values()); + } + + private appendPair( + map: Map, + noteId: string, + pair: SimilarityPair, + ): void { + let list = map.get(noteId); + if (!list) { + list = []; + map.set(noteId, list); + } + list.push(pair); + } + + /** Deterministic ordered key for an undirected note pair. */ + private makePairKey(a: string, b: string): string { + return a < b ? `${a}::${b}` : `${b}::${a}`; + } + + /** Maps each note to its tags, excluding organizational tags shared by too much of the vault to be a meaningful signal. */ + private buildTagMap(notes: Note[]): Map> { + const organizationalTags = this.findOrganizationalTags(notes); + + const map = new Map>(); + for (const n of notes) { + const meaningfulTags = (n.tags ?? []).filter(t => !organizationalTags.has(t)); + map.set(n.id, new Set(meaningfulTags)); + } + return map; + } + + private findOrganizationalTags(notes: Note[]): Set { + const tagCounts = new Map(); + for (const n of notes) { + for (const t of n.tags ?? []) { + tagCounts.set(t, (tagCounts.get(t) ?? 0) + 1); + } + } + + const threshold = notes.length * ORGANIZATIONAL_TAG_RATIO; + const organizational = new Set(); + for (const [tag, count] of tagCounts) { + if (count > threshold) organizational.add(tag); + } + return organizational; + } + + private buildLinkSet(notes: Note[]): Set { + const set = new Set(); + for (const n of notes) { + for (const link of n.links ?? []) { + set.add(this.makePairKey(n.id, link)); + } + } + return set; + } +} diff --git a/src/services/similarity/ThresholdPresets.ts b/src/services/similarity/ThresholdPresets.ts new file mode 100644 index 0000000..c7e2ed7 --- /dev/null +++ b/src/services/similarity/ThresholdPresets.ts @@ -0,0 +1,26 @@ +/** Only a direct link bypasses this floor — tags alone can never create an edge below it. */ +export const SEMANTIC_FLOOR = 0.3; + +export const DEFAULT_THRESHOLD = 0.5; + +export const TOP_K = 5; + +/** Vault size above which joplin.ai.search() is used instead of O(n²) cosine. */ +export const LARGE_VAULT_THRESHOLD = 300; + +/** Scaled by Jaccard tag overlap between two notes. */ +export const TAG_BONUS = 0.1; + +/** Smaller than TAG_BONUS on purpose — a link already renders its own edge via EdgeFactory, so this only affects redundant semantic edges. */ +export const LINK_BONUS = 0.05; + +/** Tags on more than this fraction of notes (e.g. "inbox") are excluded as organizational noise. */ +export const ORGANIZATIONAL_TAG_RATIO = 0.3; + +/** Score boost when two notes were created within a day of each other. */ +export const TEMPORAL_BONUS_1_DAY = 0.1; + +/** Score boost when two notes were created within a week of each other. */ +export const TEMPORAL_BONUS_7_DAYS = 0.05; + +export const MS_PER_DAY = 1000 * 60 * 60 * 24; From 7695e2a71a37445d6c246516b5a84193fabb7b8b Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Sat, 18 Jul 2026 23:25:40 +0530 Subject: [PATCH 19/28] ANG-008:bug fixes and format --- src/data/Database/VectorDatabase.ts | 18 ++- src/data/Database/VectorRepository.test.ts | 28 +++- src/data/Database/VectorRepository.ts | 54 +++++-- src/services/embeddings/Orchestrator.test.ts | 142 +++++++++++++----- src/services/embeddings/Orchestrator.ts | 37 ++--- .../embeddings/ProviderResolver.test.ts | 8 +- src/services/embeddings/ProviderResolver.ts | 5 +- .../Providers/JoplinNativeProvider.test.ts | 12 +- .../Providers/JoplinNativeProvider.ts | 12 +- src/services/graph/GraphBuilder.test.ts | 42 +++--- src/services/graph/GraphBuilder.ts | 4 +- src/services/similarity/EdgeFactory.test.ts | 30 +--- .../similarity/SimilarityEngine.test.ts | 137 +++++++++-------- src/services/similarity/SimilarityEngine.ts | 93 +++++++++--- 14 files changed, 409 insertions(+), 213 deletions(-) diff --git a/src/data/Database/VectorDatabase.ts b/src/data/Database/VectorDatabase.ts index c6780f4..598bc31 100644 --- a/src/data/Database/VectorDatabase.ts +++ b/src/data/Database/VectorDatabase.ts @@ -8,7 +8,11 @@ export interface IVectorDatabase { interface Sqlite3Database { run(sql: string, params: unknown[], callback: (err: Error | null) => void): void; - all(sql: string, params: unknown[], callback: (err: Error | null, rows: unknown[]) => void): void; + all( + sql: string, + params: unknown[], + callback: (err: Error | null, rows: unknown[]) => void + ): void; } /** @@ -31,11 +35,19 @@ export class VectorDatabase implements IVectorDatabase { private db: Sqlite3Database | null = null; private opening: Promise | null = null; - /** Opens (creating if needed) the vector cache database. Safe to call repeatedly. */ + /** + * Opens (creating if needed) the vector cache database. Safe to call + * repeatedly. A failed open is not cached: `opening` is reset on + * rejection so a later call can retry (e.g. after a transient lock), + * instead of every future open() re-awaiting the same stale rejection. + */ public async open(): Promise { if (this.db) return; if (!this.opening) { - this.opening = this.openInternal(); + this.opening = this.openInternal().catch((e) => { + this.opening = null; + throw e; + }); } await this.opening; } diff --git a/src/data/Database/VectorRepository.test.ts b/src/data/Database/VectorRepository.test.ts index 4b50115..177154b 100644 --- a/src/data/Database/VectorRepository.test.ts +++ b/src/data/Database/VectorRepository.test.ts @@ -10,7 +10,10 @@ import { IVectorDatabase } from './VectorDatabase'; class FakeVectorDatabase implements IVectorDatabase { public opened = false; public allCallBatchSizes: number[] = []; - private rows = new Map(); + private rows = new Map< + string, + { note_id: string; model_id: string; updated_time: number; vector: Buffer } + >(); public async open(): Promise { this.opened = true; @@ -21,13 +24,20 @@ class FakeVectorDatabase implements IVectorDatabase { return; // BEGIN TRANSACTION / COMMIT / ROLLBACK } const [noteId, modelId, updatedTime, vector] = params as [string, string, number, Buffer]; - this.rows.set(noteId, { note_id: noteId, model_id: modelId, updated_time: updatedTime, vector }); + this.rows.set(noteId, { + note_id: noteId, + model_id: modelId, + updated_time: updatedTime, + vector, + }); } public async all(_sql: string, params: unknown[]): Promise { const ids = params as string[]; this.allCallBatchSizes.push(ids.length); - const found = ids.map(id => this.rows.get(id)).filter((r): r is NonNullable => !!r); + const found = ids + .map((id) => this.rows.get(id)) + .filter((r): r is NonNullable => !!r); return found as unknown as T[]; } } @@ -74,8 +84,12 @@ describe('VectorRepository', () => { }); it('overwrites the previous entry for the same note ID', async () => { - await repo.saveMany([{ noteId: 'n1', vector: [1, 0], modelId: 'm1', updatedTime: 100 }]); - await repo.saveMany([{ noteId: 'n1', vector: [0, 1], modelId: 'm2', updatedTime: 200 }]); + await repo.saveMany([ + { noteId: 'n1', vector: [1, 0], modelId: 'm1', updatedTime: 100 }, + ]); + await repo.saveMany([ + { noteId: 'n1', vector: [0, 1], modelId: 'm2', updatedTime: 200 }, + ]); const result = await repo.getMany(['n1']); const entry = result.get('n1'); @@ -106,10 +120,10 @@ describe('VectorRepository', () => { }); describe('large vaults', () => { - it('chunks getMany so a single query never exceeds SQLite\'s bound-parameter limit', async () => { + it("chunks getMany so a single query never exceeds SQLite's bound-parameter limit", async () => { const noteIds = Array.from({ length: 1200 }, (_, i) => `n${i}`); await repo.saveMany( - noteIds.map(id => ({ noteId: id, vector: [1, 0], modelId: 'm1', updatedTime: 1 })), + noteIds.map((id) => ({ noteId: id, vector: [1, 0], modelId: 'm1', updatedTime: 1 })) ); const result = await repo.getMany(noteIds); diff --git a/src/data/Database/VectorRepository.ts b/src/data/Database/VectorRepository.ts index 4757521..55c2127 100644 --- a/src/data/Database/VectorRepository.ts +++ b/src/data/Database/VectorRepository.ts @@ -35,6 +35,12 @@ export class VectorRepository implements VectorCache { /** SQLite caps bound parameters per statement (as low as 999 on some builds); stay well under it. */ private static readonly QUERY_BATCH_SIZE = 500; + /** + * Serializes writes: sqlite transactions live on the shared connection, so + * two interleaved saveMany calls would nest BEGIN TRANSACTION and error. + */ + private writeLock: Promise = Promise.resolve(); + public constructor(private readonly db: IVectorDatabase = new VectorDatabase()) {} /** Returns cached vectors for the given note IDs, keyed by note ID. Missing notes are omitted. */ @@ -59,12 +65,22 @@ export class VectorRepository implements VectorCache { return result; } - /** Inserts or updates vectors for the given notes, in a single transaction. */ - public async saveMany(entries: VectorCacheEntry[]): Promise { + /** Inserts or updates vectors for the given notes, in a single transaction. Calls are serialized. */ + public saveMany(entries: VectorCacheEntry[]): Promise { if (entries.length === 0) { - return; + return Promise.resolve(); } + const task = this.writeLock.then(() => this.saveManyInternal(entries)); + // Keep the lock chain alive whether this write succeeds or fails. + this.writeLock = task.then( + () => undefined, + () => undefined + ); + return task; + } + + private async saveManyInternal(entries: VectorCacheEntry[]): Promise { await this.db.open(); await this.db.run('BEGIN TRANSACTION', []); @@ -77,12 +93,23 @@ export class VectorRepository implements VectorCache { model_id = excluded.model_id, updated_time = excluded.updated_time, vector = excluded.vector`, - [entry.noteId, entry.modelId, entry.updatedTime, this.encodeVector(entry.vector)], + [ + entry.noteId, + entry.modelId, + entry.updatedTime, + this.encodeVector(entry.vector), + ] ); } await this.db.run('COMMIT', []); } catch (e) { - await this.db.run('ROLLBACK', []); + // A failed ROLLBACK (e.g. "database is locked") must not mask the + // original write error. + try { + await this.db.run('ROLLBACK', []); + } catch (rollbackError) { + console.error('Vector cache rollback failed after a write error:', rollbackError); + } throw e; } } @@ -91,7 +118,7 @@ export class VectorRepository implements VectorCache { const placeholders = noteIds.map(() => '?').join(','); return this.db.all( `SELECT note_id, model_id, updated_time, vector FROM note_vectors WHERE note_id IN (${placeholders})`, - noteIds, + noteIds ); } @@ -109,13 +136,14 @@ export class VectorRepository implements VectorCache { return Buffer.from(floats.buffer, floats.byteOffset, floats.byteLength); } - /** Decodes a Float32 BLOB back into a plain number array. */ + /** + * Decodes a Float32 BLOB back into a plain number array. Copies the bytes + * first: Node pools small Buffers at arbitrary byte offsets, and viewing + * an unaligned offset with `new Float32Array(buffer, byteOffset, …)` + * throws a RangeError. + */ private decodeVector(blob: Buffer): number[] { - const floats = new Float32Array( - blob.buffer, - blob.byteOffset, - blob.byteLength / Float32Array.BYTES_PER_ELEMENT, - ); - return Array.from(floats); + const copy = blob.buffer.slice(blob.byteOffset, blob.byteOffset + blob.byteLength); + return Array.from(new Float32Array(copy)); } } diff --git a/src/services/embeddings/Orchestrator.test.ts b/src/services/embeddings/Orchestrator.test.ts index 366d4c6..85cb386 100644 --- a/src/services/embeddings/Orchestrator.test.ts +++ b/src/services/embeddings/Orchestrator.test.ts @@ -44,7 +44,10 @@ describe('EmbeddingOrchestrator', () => { fetchVectorsByNoteIds: jest.fn().mockResolvedValue(mockVectors), }); - const notes = [makeNote('n1', 'Title 1', 'Body 1'), makeNote('n2', 'Title 2', 'Body 2')]; + const notes = [ + makeNote('n1', 'Title 1', 'Body 1'), + makeNote('n2', 'Title 2', 'Body 2'), + ]; const result = await orchestrator.embedNotes(notes); expect(result.embeddedNotes).toHaveLength(2); @@ -89,10 +92,7 @@ describe('EmbeddingOrchestrator', () => { fetchVectorsByNoteIds: jest.fn().mockResolvedValue(mockVectors), }); - await orchestrator.embedNotes([ - makeNote('n1', 'T1', 'B1'), - makeNote('n2', 'T2', 'B2'), - ]); + await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1'), makeNote('n2', 'T2', 'B2')]); expect(progressUpdates).toEqual([ { current: 0, total: 2 }, @@ -132,11 +132,17 @@ describe('EmbeddingOrchestrator', () => { describe('vector caching', () => { it('serves an unchanged, same-model note from the cache without calling the provider', async () => { const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map()); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm1', + fetchVectorsByNoteIds, + }); orchestrator.setCache({ - getMany: jest.fn().mockResolvedValue( - new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]), - ), + getMany: jest + .fn() + .mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]) + ), saveMany: jest.fn().mockResolvedValue(undefined), }); @@ -148,12 +154,20 @@ describe('EmbeddingOrchestrator', () => { }); it('re-fetches a note whose updated_time no longer matches the cached entry', async () => { - const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.9, 0.9]]])); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + const fetchVectorsByNoteIds = jest + .fn() + .mockResolvedValue(new Map([['n1', [0.9, 0.9]]])); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm1', + fetchVectorsByNoteIds, + }); orchestrator.setCache({ - getMany: jest.fn().mockResolvedValue( - new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]), - ), + getMany: jest + .fn() + .mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]) + ), saveMany: jest.fn().mockResolvedValue(undefined), }); @@ -168,11 +182,17 @@ describe('EmbeddingOrchestrator', () => { // queued for re-fetch — but Joplin's AI index hasn't caught up yet and // returns nothing for it. The stale pre-edit vector must not be used. const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map()); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm1', + fetchVectorsByNoteIds, + }); orchestrator.setCache({ - getMany: jest.fn().mockResolvedValue( - new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]), - ), + getMany: jest + .fn() + .mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]) + ), saveMany: jest.fn().mockResolvedValue(undefined), }); @@ -180,16 +200,26 @@ describe('EmbeddingOrchestrator', () => { expect(fetchVectorsByNoteIds).toHaveBeenCalledWith(['n1']); expect(result.embeddedNotes).toHaveLength(0); - expect(result.errors).toEqual([{ noteId: 'n1', error: 'Note not yet indexed by Joplin AI.' }]); + expect(result.errors).toEqual([ + { noteId: 'n1', error: 'Note not yet indexed by Joplin AI.' }, + ]); }); it('re-fetches a note whose cached entry belongs to a different embedding model', async () => { - const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.9, 0.9]]])); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm2', fetchVectorsByNoteIds }); + const fetchVectorsByNoteIds = jest + .fn() + .mockResolvedValue(new Map([['n1', [0.9, 0.9]]])); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm2', + fetchVectorsByNoteIds, + }); orchestrator.setCache({ - getMany: jest.fn().mockResolvedValue( - new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]), - ), + getMany: jest + .fn() + .mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 50 }]]) + ), saveMany: jest.fn().mockResolvedValue(undefined), }); @@ -200,12 +230,20 @@ describe('EmbeddingOrchestrator', () => { }); it('only asks the provider for stale/missing notes, mixing in cache hits', async () => { - const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n2', [0.4, 0.5]]])); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + const fetchVectorsByNoteIds = jest + .fn() + .mockResolvedValue(new Map([['n2', [0.4, 0.5]]])); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm1', + fetchVectorsByNoteIds, + }); orchestrator.setCache({ - getMany: jest.fn().mockResolvedValue( - new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 10 }]]), - ), + getMany: jest + .fn() + .mockResolvedValue( + new Map([['n1', { vector: [0.1, 0.2], modelId: 'm1', updatedTime: 10 }]]) + ), saveMany: jest.fn().mockResolvedValue(undefined), }); @@ -220,9 +258,15 @@ describe('EmbeddingOrchestrator', () => { }); it('saves freshly fetched vectors back to the cache with the note updated_time and model', async () => { - const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); + const fetchVectorsByNoteIds = jest + .fn() + .mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); const saveMany = jest.fn().mockResolvedValue(undefined); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm1', + fetchVectorsByNoteIds, + }); orchestrator.setCache({ getMany: jest.fn().mockResolvedValue(new Map()), saveMany }); await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 42)]); @@ -235,7 +279,11 @@ describe('EmbeddingOrchestrator', () => { it('does not save notes the provider failed to return', async () => { const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map()); const saveMany = jest.fn().mockResolvedValue(undefined); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm1', + fetchVectorsByNoteIds, + }); orchestrator.setCache({ getMany: jest.fn().mockResolvedValue(new Map()), saveMany }); await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 42)]); @@ -244,8 +292,14 @@ describe('EmbeddingOrchestrator', () => { }); it('falls back to a full fetch when the cache read throws', async () => { - const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + const fetchVectorsByNoteIds = jest + .fn() + .mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm1', + fetchVectorsByNoteIds, + }); orchestrator.setCache({ getMany: jest.fn().mockRejectedValue(new Error('disk error')), saveMany: jest.fn().mockResolvedValue(undefined), @@ -258,8 +312,14 @@ describe('EmbeddingOrchestrator', () => { }); it('still returns results when the cache write throws', async () => { - const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + const fetchVectorsByNoteIds = jest + .fn() + .mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm1', + fetchVectorsByNoteIds, + }); orchestrator.setCache({ getMany: jest.fn().mockResolvedValue(new Map()), saveMany: jest.fn().mockRejectedValue(new Error('disk full')), @@ -272,8 +332,14 @@ describe('EmbeddingOrchestrator', () => { }); it('behaves exactly as before when no cache is set', async () => { - const fetchVectorsByNoteIds = jest.fn().mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); - orchestrator.setProvider({ id: 'joplin-native', modelName: 'm1', fetchVectorsByNoteIds }); + const fetchVectorsByNoteIds = jest + .fn() + .mockResolvedValue(new Map([['n1', [0.4, 0.5]]])); + orchestrator.setProvider({ + id: 'joplin-native', + modelName: 'm1', + fetchVectorsByNoteIds, + }); const result = await orchestrator.embedNotes([makeNote('n1', 'T1', 'B1', 42)]); diff --git a/src/services/embeddings/Orchestrator.ts b/src/services/embeddings/Orchestrator.ts index 130ff9a..f1fc4c8 100644 --- a/src/services/embeddings/Orchestrator.ts +++ b/src/services/embeddings/Orchestrator.ts @@ -1,11 +1,6 @@ import { Note } from '../../data/Types'; import { CachedVector, VectorCache, VectorCacheEntry } from '../../data/Database/VectorRepository'; -import { - EmbeddingProvider, - EmbeddedNote, - EmbeddingResult, - BatchProgress, -} from './Types'; +import { EmbeddingProvider, EmbeddedNote, EmbeddingResult, BatchProgress } from './Types'; export class EmbeddingOrchestrator { private provider: EmbeddingProvider | null = null; @@ -37,7 +32,7 @@ export class EmbeddingOrchestrator { } if (!this.provider) { - const errors = notes.map(n => ({ noteId: n.id, error: 'No provider configured' })); + const errors = notes.map((n) => ({ noteId: n.id, error: 'No provider configured' })); return { embeddedNotes: [], errors }; } @@ -64,7 +59,7 @@ export class EmbeddingOrchestrator { return { embeddedNotes, errors }; } catch (e) { const msg = e instanceof Error ? e.message : String(e); - const errors = notes.map(n => ({ noteId: n.id, error: msg })); + const errors = notes.map((n) => ({ noteId: n.id, error: msg })); return { embeddedNotes: [], errors }; } } @@ -74,15 +69,21 @@ export class EmbeddingOrchestrator { * `updated_time` and model haven't changed and only asking the provider * to (re-)fetch the rest. */ - private async resolveVectors(notes: Note[], provider: EmbeddingProvider): Promise> { + private async resolveVectors( + notes: Note[], + provider: EmbeddingProvider + ): Promise> { const modelId = provider.modelName; const cached = await this.getCachedVectors(notes); - const notesToFetch = notes.filter(n => !this.isFreshCacheHit(cached.get(n.id), n, modelId)); + const notesToFetch = notes.filter( + (n) => !this.isFreshCacheHit(cached.get(n.id), n, modelId) + ); - const fresh = notesToFetch.length > 0 - ? await provider.fetchVectorsByNoteIds(notesToFetch.map(n => n.id)) - : new Map(); + const fresh = + notesToFetch.length > 0 + ? await provider.fetchVectorsByNoteIds(notesToFetch.map((n) => n.id)) + : new Map(); await this.saveFreshVectors(notesToFetch, fresh, modelId); @@ -94,7 +95,7 @@ export class EmbeddingOrchestrator { notes: Note[], cached: Map, fresh: Map, - modelId: string, + modelId: string ): Map { const merged = new Map(); for (const note of notes) { @@ -115,7 +116,7 @@ export class EmbeddingOrchestrator { private async getCachedVectors(notes: Note[]): Promise> { if (!this.cache) return new Map(); try { - return await this.cache.getMany(notes.map(n => n.id)); + return await this.cache.getMany(notes.map((n) => n.id)); } catch (e) { console.error('Vector cache read failed, falling back to a full fetch:', e); return new Map(); @@ -125,13 +126,13 @@ export class EmbeddingOrchestrator { private async saveFreshVectors( notes: Note[], vectors: Map, - modelId: string, + modelId: string ): Promise { if (!this.cache || vectors.size === 0) return; const entries: VectorCacheEntry[] = notes - .filter(n => vectors.has(n.id)) - .map(n => ({ + .filter((n) => vectors.has(n.id)) + .map((n) => ({ noteId: n.id, vector: vectors.get(n.id)!, modelId, diff --git a/src/services/embeddings/ProviderResolver.test.ts b/src/services/embeddings/ProviderResolver.test.ts index 9d9ad58..26c7fa6 100644 --- a/src/services/embeddings/ProviderResolver.test.ts +++ b/src/services/embeddings/ProviderResolver.test.ts @@ -39,7 +39,9 @@ describe('ProviderResolver', () => { it('returns provider when index is ready', async () => { (joplin as any).ai = { - getIndexStatus: jest.fn().mockResolvedValue({ ready: true, state: 'ready', modelId: 'test-model' }), + getIndexStatus: jest + .fn() + .mockResolvedValue({ ready: true, state: 'ready', modelId: 'test-model' }), getEmbeddings: jest.fn(), }; const provider = await ProviderResolver.resolveWithValidation(); @@ -49,7 +51,9 @@ describe('ProviderResolver', () => { it('returns provider while the index is still indexing, since search still works with partial data', async () => { (joplin as any).ai = { - getIndexStatus: jest.fn().mockResolvedValue({ ready: false, state: 'indexing', modelId: 'test-model' }), + getIndexStatus: jest + .fn() + .mockResolvedValue({ ready: false, state: 'indexing', modelId: 'test-model' }), getEmbeddings: jest.fn(), }; const provider = await ProviderResolver.resolveWithValidation(); diff --git a/src/services/embeddings/ProviderResolver.ts b/src/services/embeddings/ProviderResolver.ts index 446d52b..17371ea 100644 --- a/src/services/embeddings/ProviderResolver.ts +++ b/src/services/embeddings/ProviderResolver.ts @@ -3,7 +3,6 @@ import { EmbeddingProvider, ProviderConfig } from './Types'; import { JoplinNativeProvider, JoplinAiApi, isIndexUsable } from './providers/JoplinNativeProvider'; export class ProviderResolver { - /** * Resolves the native embedding provider after verifying that Joplin AI is * available and its embedding index is usable. @@ -11,7 +10,9 @@ export class ProviderResolver { public static async resolveWithValidation(): Promise { const joplinAi = joplin.ai as unknown as JoplinAiApi | undefined; if (!joplinAi) { - throw new Error('joplin.ai is not available. Enable AI in Settings → AI. Requires Joplin v3.7+.'); + throw new Error( + 'joplin.ai is not available. Enable AI in Settings → AI. Requires Joplin v3.7+.' + ); } const status = await joplinAi.getIndexStatus(); if (!status || !isIndexUsable(status.state)) { diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.test.ts b/src/services/embeddings/Providers/JoplinNativeProvider.test.ts index 99a773b..8e8d67f 100644 --- a/src/services/embeddings/Providers/JoplinNativeProvider.test.ts +++ b/src/services/embeddings/Providers/JoplinNativeProvider.test.ts @@ -30,7 +30,11 @@ describe('JoplinNativeProvider', () => { getEmbeddings: jest.Mock; }; - ai.getIndexStatus.mockResolvedValue({ ready: true, state: 'ready', modelId: 'fresh-model' }); + ai.getIndexStatus.mockResolvedValue({ + ready: true, + state: 'ready', + modelId: 'fresh-model', + }); ai.getEmbeddings.mockResolvedValue({ modelId: 'fresh-model', dimension: 2, @@ -125,7 +129,11 @@ describe('JoplinNativeProvider', () => { getEmbeddings: jest.Mock; }; - ai.getIndexStatus.mockResolvedValue({ ready: false, state: 'indexing', modelId: 'test-model' }); + ai.getIndexStatus.mockResolvedValue({ + ready: false, + state: 'indexing', + modelId: 'test-model', + }); ai.getEmbeddings.mockResolvedValue({ modelId: 'test-model', dimension: 2, diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.ts b/src/services/embeddings/Providers/JoplinNativeProvider.ts index 9a13758..8ee2aae 100644 --- a/src/services/embeddings/Providers/JoplinNativeProvider.ts +++ b/src/services/embeddings/Providers/JoplinNativeProvider.ts @@ -42,7 +42,11 @@ export interface JoplinAiApi { getEmbeddings: (options: GetEmbeddingsOptions) => Promise; } -const BLOCKING_STATES: ReadonlySet = new Set(['unavailable', 'disabled', 'preparing']); +const BLOCKING_STATES: ReadonlySet = new Set([ + 'unavailable', + 'disabled', + 'preparing', +]); /** True once the index has enough data to fetch from, even if still indexing. */ export function isIndexUsable(state: AiIndexState | undefined): boolean { @@ -119,7 +123,7 @@ export class JoplinNativeProvider implements EmbeddingProvider { */ private async fetchAllPages( api: JoplinAiApi, - noteIds: string[], + noteIds: string[] ): Promise> { let trackedModelId = await this.requireUsableIndex(api); @@ -130,7 +134,9 @@ export class JoplinNativeProvider implements EmbeddingProvider { while (true) { if (pageCount >= JoplinNativeProvider.MAX_PAGES) { - throw new Error('Too many pages. The embedding index may be in an unexpected state.'); + throw new Error( + 'Too many pages. The embedding index may be in an unexpected state.' + ); } pageCount++; diff --git a/src/services/graph/GraphBuilder.test.ts b/src/services/graph/GraphBuilder.test.ts index 0864f56..1d4c721 100644 --- a/src/services/graph/GraphBuilder.test.ts +++ b/src/services/graph/GraphBuilder.test.ts @@ -9,11 +9,7 @@ jest.mock('../similarity/SimilarityEngine'); const MockEdgeFactory = EdgeFactory as jest.MockedClass; const MockSimilarityEngine = SimilarityEngine as jest.MockedClass; -function note( - id: string, - title: string, - links: string[] = [] -): Note { +function note(id: string, title: string, links: string[] = []): Note { return { id, parent_id: 'p1', @@ -45,9 +41,7 @@ describe('GraphBuilder', () => { }); it('computes degree from edges', () => { - mockEdgeFactory.createEdges.mockReturnValue([ - { source: 'a', target: 'b', type: 'link' }, - ]); + mockEdgeFactory.createEdges.mockReturnValue([{ source: 'a', target: 'b', type: 'link' }]); const notes = [note('a', 'A'), note('b', 'B')]; const result = builder.build(notes); expect(result.nodes[0].data.degree).toBe(1); @@ -83,13 +77,20 @@ describe('GraphBuilder', () => { describe('buildWithSimilarity', () => { it('adds semantic edges computed from embeddings alongside structural edges', async () => { - mockEdgeFactory.createEdges.mockReturnValue([{ source: 'a', target: 'c', type: 'link' }]); + mockEdgeFactory.createEdges.mockReturnValue([ + { source: 'a', target: 'c', type: 'link' }, + ]); mockEdgeFactory.createSemanticEdges.mockReturnValue([ { source: 'a', target: 'b', type: 'semantic' }, ]); - MockSimilarityEngine.mockImplementation(() => ({ - compute: jest.fn().mockResolvedValue([{ source: 'a', target: 'b', score: 0.8 }]), - }) as unknown as SimilarityEngine); + MockSimilarityEngine.mockImplementation( + () => + ({ + compute: jest + .fn() + .mockResolvedValue([{ source: 'a', target: 'b', score: 0.8 }]), + } as unknown as SimilarityEngine) + ); const notes = [note('a', 'A'), note('b', 'B'), note('c', 'C')]; const embeddedNotes = [ @@ -102,17 +103,24 @@ describe('GraphBuilder', () => { expect(mockEdgeFactory.createSemanticEdges).toHaveBeenCalledWith([ { source: 'a', target: 'b', score: 0.8 }, ]); - expect(result.edges).toContainEqual({ data: { source: 'a', target: 'b', type: 'semantic' } }); - expect(result.edges).toContainEqual({ data: { source: 'a', target: 'c', type: 'link' } }); + expect(result.edges).toContainEqual({ + data: { source: 'a', target: 'b', type: 'semantic' }, + }); + expect(result.edges).toContainEqual({ + data: { source: 'a', target: 'c', type: 'link' }, + }); expect(result.edges).toHaveLength(2); }); it('still returns a graph when there are no semantic matches', async () => { mockEdgeFactory.createEdges.mockReturnValue([]); mockEdgeFactory.createSemanticEdges.mockReturnValue([]); - MockSimilarityEngine.mockImplementation(() => ({ - compute: jest.fn().mockResolvedValue([]), - }) as unknown as SimilarityEngine); + MockSimilarityEngine.mockImplementation( + () => + ({ + compute: jest.fn().mockResolvedValue([]), + } as unknown as SimilarityEngine) + ); const notes = [note('a', 'A'), note('b', 'B')]; const result = await builder.buildWithSimilarity(notes, []); diff --git a/src/services/graph/GraphBuilder.ts b/src/services/graph/GraphBuilder.ts index 88f1526..85aa096 100644 --- a/src/services/graph/GraphBuilder.ts +++ b/src/services/graph/GraphBuilder.ts @@ -27,7 +27,7 @@ export class GraphBuilder { */ public async buildWithSimilarity( notes: Note[], - embeddedNotes: EmbeddedNote[], + embeddedNotes: EmbeddedNote[] ): Promise { const structuralEdges = this.edgeFactory.createEdges(notes); @@ -92,7 +92,7 @@ export class GraphBuilder { private logGraphStats( nodes: Array<{ data: GraphNode }>, visibleEdges: GraphEdge[], - degreeMap: Map, + degreeMap: Map ): void { const connectedIds = new Set(); for (const edge of visibleEdges) { diff --git a/src/services/similarity/EdgeFactory.test.ts b/src/services/similarity/EdgeFactory.test.ts index c753663..61ef6ee 100644 --- a/src/services/similarity/EdgeFactory.test.ts +++ b/src/services/similarity/EdgeFactory.test.ts @@ -1,12 +1,7 @@ import { EdgeFactory } from './EdgeFactory'; import { Note } from '../../data/Types'; -function note( - id: string, - title: string, - links: string[] = [], - tags: string[] = [] -): Note { +function note(id: string, title: string, links: string[] = [], tags: string[] = []): Note { return { id, parent_id: 'p1', @@ -35,26 +30,17 @@ describe('EdgeFactory', () => { }); it('ignores resource links that are not note IDs', () => { - const notes = [ - note('a', 'A', ['resource123']), - note('b', 'B', ['resource123']), - ]; + const notes = [note('a', 'A', ['resource123']), note('b', 'B', ['resource123'])]; expect(factory.createEdges(notes)).toEqual([]); }); it('creates link edge when note body references another note ID', () => { - const edges = factory.createEdges([ - note('a', 'A', ['b']), - note('b', 'B', []), - ]); + const edges = factory.createEdges([note('a', 'A', ['b']), note('b', 'B', [])]); expect(edges).toEqual([{ source: 'a', target: 'b', type: 'link' }]); }); it('creates bidirectional links when notes reference each other', () => { - const edges = factory.createEdges([ - note('a', 'A', ['b']), - note('b', 'B', ['a']), - ]); + const edges = factory.createEdges([note('a', 'A', ['b']), note('b', 'B', ['a'])]); expect(edges).toHaveLength(2); expect(edges).toContainEqual({ source: 'a', target: 'b', type: 'link' }); expect(edges).toContainEqual({ source: 'b', target: 'a', type: 'link' }); @@ -65,9 +51,7 @@ describe('EdgeFactory', () => { note('a', 'A', [], ['shared']), note('b', 'B', [], ['shared']), ]); - expect(edges).toEqual([ - { source: 'a', target: 'b', type: 'tag', tagName: 'shared' }, - ]); + expect(edges).toEqual([{ source: 'a', target: 'b', type: 'tag', tagName: 'shared' }]); }); it('merges multiple shared tag names into one edge', () => { @@ -108,9 +92,7 @@ describe('EdgeFactory', () => { }); it('creates a semantic edge for each positive-score pair', () => { - const edges = factory.createSemanticEdges([ - { source: 'a', target: 'b', score: 0.8 }, - ]); + const edges = factory.createSemanticEdges([{ source: 'a', target: 'b', score: 0.8 }]); expect(edges).toEqual([{ source: 'a', target: 'b', type: 'semantic' }]); }); diff --git a/src/services/similarity/SimilarityEngine.test.ts b/src/services/similarity/SimilarityEngine.test.ts index f9c8f1c..a94d374 100644 --- a/src/services/similarity/SimilarityEngine.test.ts +++ b/src/services/similarity/SimilarityEngine.test.ts @@ -7,7 +7,7 @@ function makeNote( title: string, links: string[] = [], tags: string[] = [], - createdTime = 0, + createdTime = 0 ): Note { return { id, @@ -36,10 +36,7 @@ describe('SimilarityEngine', () => { }); it('returns empty for a single note', async () => { - const engine = new SimilarityEngine( - [makeNote('a', 'A')], - [embed('a', [1, 0, 0])], - ); + const engine = new SimilarityEngine([makeNote('a', 'A')], [embed('a', [1, 0, 0])]); const pairs = await engine.compute(); expect(pairs).toEqual([]); }); @@ -83,9 +80,7 @@ describe('SimilarityEngine', () => { const engine = new SimilarityEngine(notes, embedded); const pairs = await engine.compute(); - const involvesB = pairs.some( - p => p.source === 'b' || p.target === 'b', - ); + const involvesB = pairs.some((p) => p.source === 'b' || p.target === 'b'); expect(involvesB).toBe(false); }); }); @@ -93,11 +88,7 @@ describe('SimilarityEngine', () => { describe('normalization', () => { it('produces scores in [0, 1] range', async () => { const notes = [makeNote('a', 'A'), makeNote('b', 'B'), makeNote('c', 'C')]; - const embedded = [ - embed('a', [1, 0]), - embed('b', [0.95, 0.3]), - embed('c', [0.3, 0.95]), - ]; + const embedded = [embed('a', [1, 0]), embed('b', [0.95, 0.3]), embed('c', [0.3, 0.95])]; const engine = new SimilarityEngine(notes, embedded); const pairs = await engine.compute(); @@ -109,30 +100,29 @@ describe('SimilarityEngine', () => { }); it('gives higher scores to more similar notes', async () => { - const notes = [ - makeNote('a', 'A'), makeNote('b', 'B'), - makeNote('c', 'C'), makeNote('d', 'D'), - ]; + // With the floor applied to the raw score before normalize, whichever + // pair is weakest among the floor survivors normalizes to exactly 0 — + // b-c (raw ~0.589) plays that role here so it doesn't drag a-c down + // with it, letting both a-b and a-c clear the threshold with a-b + // still scoring higher. + const notes = [makeNote('a', 'A'), makeNote('b', 'B'), makeNote('c', 'C')]; const embedded = [ - embed('a', [1, 0]), - embed('b', [0.95, 0.31]), - embed('c', [0.7, 0.71]), - embed('d', [-1, 0]), + embed('a', [1, 0, 0]), + embed('b', [0.9, Math.sqrt(1 - 0.81), 0]), + embed('c', [0.8, -0.3, Math.sqrt(0.27)]), ]; const engine = new SimilarityEngine(notes, embedded); const pairs = await engine.compute(); const abScore = pairs.find( - p => - (p.source === 'a' && p.target === 'b') || - (p.source === 'b' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a') ); const acScore = pairs.find( - p => - (p.source === 'a' && p.target === 'c') || - (p.source === 'c' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'c') || (p.source === 'c' && p.target === 'a') ); expect(abScore).toBeDefined(); @@ -168,14 +158,12 @@ describe('SimilarityEngine', () => { const pairs = await engine.compute(); const abWithTag = pairs.find( - p => - (p.source === 'a' && p.target === 'b') || - (p.source === 'b' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a') ); const acNoTag = pairs.find( - p => - (p.source === 'a' && p.target === 'c') || - (p.source === 'c' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'c') || (p.source === 'c' && p.target === 'a') ); expect(abWithTag).toBeDefined(); @@ -216,10 +204,12 @@ describe('SimilarityEngine', () => { const pairs = await engine.compute(); const ab = pairs.find( - p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a') ); const cd = pairs.find( - p => (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c'), + (p) => + (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c') ); expect(ab).toBeDefined(); @@ -259,10 +249,12 @@ describe('SimilarityEngine', () => { const pairs = await engine.compute(); const abSharesProject = pairs.find( - p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a') ); const acSharesOnlyInbox = pairs.find( - p => (p.source === 'a' && p.target === 'c') || (p.source === 'c' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'c') || (p.source === 'c' && p.target === 'a') ); expect(abSharesProject).toBeDefined(); @@ -288,14 +280,12 @@ describe('SimilarityEngine', () => { const pairs = await engine.compute(); const abWithLink = pairs.find( - p => - (p.source === 'a' && p.target === 'b') || - (p.source === 'b' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a') ); const acNoLink = pairs.find( - p => - (p.source === 'a' && p.target === 'c') || - (p.source === 'c' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'c') || (p.source === 'c' && p.target === 'a') ); expect(abWithLink).toBeDefined(); @@ -337,10 +327,12 @@ describe('SimilarityEngine', () => { const pairs = await engine.compute(); const ab = pairs.find( - p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a') ); const cd = pairs.find( - p => (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c'), + (p) => + (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c') ); expect(ab).toBeDefined(); @@ -374,10 +366,12 @@ describe('SimilarityEngine', () => { const pairs = await engine.compute(); const ab = pairs.find( - p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a') ); const cd = pairs.find( - p => (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c'), + (p) => + (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c') ); expect(ab).toBeDefined(); @@ -411,10 +405,12 @@ describe('SimilarityEngine', () => { const pairs = await engine.compute(); const ab = pairs.find( - p => (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a'), + (p) => + (p.source === 'a' && p.target === 'b') || (p.source === 'b' && p.target === 'a') ); const cd = pairs.find( - p => (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c'), + (p) => + (p.source === 'c' && p.target === 'd') || (p.source === 'd' && p.target === 'c') ); expect(ab).toBeDefined(); @@ -430,10 +426,7 @@ describe('SimilarityEngine', () => { for (let i = 0; i < 6; i++) { notes.push(makeNote(`n${i}`, `Note ${i}`)); embedded.push( - embed(`n${i}`, [ - Math.cos((i * Math.PI) / 3), - Math.sin((i * Math.PI) / 3), - ]), + embed(`n${i}`, [Math.cos((i * Math.PI) / 3), Math.sin((i * Math.PI) / 3)]) ); } @@ -441,9 +434,7 @@ describe('SimilarityEngine', () => { const pairs = await engine.compute(); for (const note of notes) { - const edges = pairs.filter( - p => p.source === note.id || p.target === note.id, - ); + const edges = pairs.filter((p) => p.source === note.id || p.target === note.id); expect(edges.length).toBeLessThanOrEqual(5); } }); @@ -472,7 +463,7 @@ describe('SimilarityEngine', () => { const engine = new SimilarityEngine(notes, embedded); const pairs = await engine.compute(); - const aEdges = pairs.filter(p => p.source === 'a' || p.target === 'a'); + const aEdges = pairs.filter((p) => p.source === 'a' || p.target === 'a'); expect(aEdges.length).toBeGreaterThan(5); }); }); @@ -483,10 +474,7 @@ describe('SimilarityEngine', () => { // linked. SEMANTIC_FLOOR must reject them before bonuses or threshold // ever apply — tags alone can never manufacture an edge out of a weak // semantic score. - const notes = [ - makeNote('a', 'A', [], ['shared']), - makeNote('b', 'B', [], ['shared']), - ]; + const notes = [makeNote('a', 'A', [], ['shared']), makeNote('b', 'B', [], ['shared'])]; const embedded = [embed('a', [1, 0]), embed('b', [0, 1])]; const engine = new SimilarityEngine(notes, embedded); @@ -512,4 +500,31 @@ describe('SimilarityEngine', () => { expect(pairs).toEqual([]); }); }); + + describe('floor is applied to raw scores, before normalization', () => { + it('returns zero pairs for a vault of unrelated notes even when normalization runs', async () => { + // Raw dots span 0 to 0.12 (spread >= 0.1, so normalization would run + // and map the best pair to 1.0). Vectors are near-orthogonal with + // only a small "leakage" component on a shared axis, so every + // pairwise raw score stays below SEMANTIC_FLOOR (0.3) — with the + // floor applied on the raw scale, nothing survives. + const notes = [ + makeNote('a', 'A'), + makeNote('b', 'B'), + makeNote('c', 'C'), + makeNote('d', 'D'), + ]; + const embedded = [ + embed('a', [1, 0.05, 0, 0]), + embed('b', [0, 1, 0.08, 0]), + embed('c', [0, 0, 1, 0.12]), + embed('d', [0.03, 0, 0, 1]), + ]; + + const engine = new SimilarityEngine(notes, embedded); + const pairs = await engine.compute(); + + expect(pairs).toEqual([]); + }); + }); }); diff --git a/src/services/similarity/SimilarityEngine.ts b/src/services/similarity/SimilarityEngine.ts index dd8a839..f9ff86b 100644 --- a/src/services/similarity/SimilarityEngine.ts +++ b/src/services/similarity/SimilarityEngine.ts @@ -29,17 +29,27 @@ export class SimilarityEngine { private readonly createdTimeMap: Map; public constructor(notes: Note[], embeddedNotes: EmbeddedNote[]) { - this.noteIds = notes.map(n => n.id); + this.noteIds = notes.map((n) => n.id); this.vectors = new Map(); for (const en of embeddedNotes) { this.vectors.set(en.note.id, en.embedding); } this.tagMap = this.buildTagMap(notes); this.linkSet = this.buildLinkSet(notes); - this.createdTimeMap = new Map(notes.map(n => [n.id, n.created_time])); + this.createdTimeMap = new Map(notes.map((n) => [n.id, n.created_time])); } - /** Orchestrates the full similarity pipeline: compute → normalize → floor → enrich → threshold → top-K. */ + /** + * Orchestrates the full similarity pipeline: + * compute → floor (raw scores) → normalize → enrich → threshold → top-K. + * + * SEMANTIC_FLOOR is applied to *raw* scores, before normalization. Min-max + * normalization always maps the batch's most-similar pair to exactly 1.0, + * so a post-normalization floor can never reject it — even in a vault of + * completely unrelated notes. Flooring on the raw scale (where 0.3 has an + * absolute meaning) is what actually guarantees that tags alone can never + * manufacture an edge out of a weak semantic score. + */ public async compute(): Promise { if (this.noteIds.length <= 1) { return []; @@ -51,9 +61,14 @@ export class SimilarityEngine { return []; } - const normalized = this.normalize(rawPairs); - const aboveFloor = this.filterBelowFloor(normalized, SEMANTIC_FLOOR); - const enriched = this.addBonusPoints(aboveFloor); + const aboveFloor = this.filterBelowFloor(rawPairs, SEMANTIC_FLOOR); + + if (aboveFloor.length === 0) { + return []; + } + + const normalized = this.normalize(aboveFloor); + const enriched = this.addBonusPoints(normalized); const aboveThreshold = this.filterBelowThreshold(enriched, DEFAULT_THRESHOLD); const topPairs = this.selectTopK(aboveThreshold, TOP_K); @@ -96,6 +111,16 @@ export class SimilarityEngine { * Uses joplin.ai.search({ noteId }) to find candidate pairs via vector index. * Only checks that joplin.ai itself exists — never probes a specific method * property without invoking it (see JoplinNativeProvider.validateAiApi for why). + * + * Score-scale assumption: search relevance scores are treated as raw + * similarity scores and flow through the same floor → normalize pipeline + * as cosine scores. + * + * Failure handling: individual per-note search failures are skipped (a + * partial candidate set is still useful), but if *every* call fails — + * e.g. joplin.ai exists but search doesn't on this Joplin version — we + * fall back to O(n²) cosine instead of silently returning zero pairs. + * Retry/backoff and progress/cancel for this path are ANG-012. */ private async computeSearchPairs(): Promise { const joplinAi = joplin.ai as unknown as @@ -107,6 +132,8 @@ export class SimilarityEngine { const seen = new Set(); const pairs: SimilarityPair[] = []; + let successCount = 0; + let firstError: unknown = null; for (const noteId of this.noteIds) { try { @@ -114,6 +141,7 @@ export class SimilarityEngine { query: { noteId }, relevance: 'normal', }); + successCount++; for (const r of results) { if (!this.vectors.has(r.noteId) || r.noteId === noteId) { @@ -124,17 +152,31 @@ export class SimilarityEngine { if (seen.has(key)) continue; seen.add(key); - const [source, target] = noteId < r.noteId - ? [noteId, r.noteId] - : [r.noteId, noteId]; + const [source, target] = + noteId < r.noteId ? [noteId, r.noteId] : [r.noteId, noteId]; pairs.push({ source, target, score: r.score }); } - } catch { + } catch (e) { + if (firstError === null) { + firstError = e; + console.warn( + 'joplin.ai.search failed for a note; skipping it. First error:', + e + ); + } continue; } } + if (successCount === 0 && this.noteIds.length > 0) { + console.warn( + 'All joplin.ai.search calls failed; falling back to pairwise cosine similarity.', + firstError + ); + return this.computeCosinePairs(); + } + return pairs; } @@ -172,7 +214,8 @@ export class SimilarityEngine { /** Adds shared-tag, direct-link, and temporal-proximity bonuses to each pair's score. */ private addBonusPoints(pairs: SimilarityPair[]): SimilarityPair[] { for (const p of pairs) { - p.score += this.sharedTagBonus(p) + this.directLinkBonus(p) + this.temporalProximityBonus(p); + p.score += + this.sharedTagBonus(p) + this.directLinkBonus(p) + this.temporalProximityBonus(p); } return pairs; } @@ -213,18 +256,19 @@ export class SimilarityEngine { } /** - * Removes pairs below the safety floor — unless the notes are directly - * linked, in which case they're kept and left for the threshold check - * below. SEMANTIC_FLOOR guards against spurious tag-only edges, not - * against edges the user already created explicitly. + * Removes pairs whose *raw* score is below the safety floor — unless the + * notes are directly linked, in which case they're kept and left for the + * threshold check later. Runs before normalization on purpose: the floor + * guards against spurious tag-only edges, which requires an absolute + * scale, not a batch-relative one. */ private filterBelowFloor(pairs: SimilarityPair[], floor: number): SimilarityPair[] { - return pairs.filter(p => p.score >= floor || this.isDirectlyLinked(p)); + return pairs.filter((p) => p.score >= floor || this.isDirectlyLinked(p)); } /** Keeps only pairs whose bonus-boosted score clears the threshold. */ private filterBelowThreshold(pairs: SimilarityPair[], threshold: number): SimilarityPair[] { - return pairs.filter(p => p.score >= threshold); + return pairs.filter((p) => p.score >= threshold); } /** @@ -232,13 +276,18 @@ export class SimilarityEngine { * their union, so a note that many others pick as one of their top-K can * end up with more than K edges. This is the standard k-nearest-neighbor * graph definition and preserves degree as a centrality signal. + * Output pairs are always oriented source < target. */ private selectTopK(pairs: SimilarityPair[], k: number): SimilarityPair[] { const bySource = new Map(); for (const p of pairs) { this.appendPair(bySource, p.source, p); - this.appendPair(bySource, p.target, { source: p.target, target: p.source, score: p.score }); + this.appendPair(bySource, p.target, { + source: p.target, + target: p.source, + score: p.score, + }); } const deduped = new Map(); @@ -250,7 +299,9 @@ export class SimilarityEngine { for (const p of kept) { const key = this.makePairKey(p.source, p.target); if (!deduped.has(key)) { - deduped.set(key, p); + const [source, target] = + p.source < p.target ? [p.source, p.target] : [p.target, p.source]; + deduped.set(key, { source, target, score: p.score }); } } } @@ -261,7 +312,7 @@ export class SimilarityEngine { private appendPair( map: Map, noteId: string, - pair: SimilarityPair, + pair: SimilarityPair ): void { let list = map.get(noteId); if (!list) { @@ -282,7 +333,7 @@ export class SimilarityEngine { const map = new Map>(); for (const n of notes) { - const meaningfulTags = (n.tags ?? []).filter(t => !organizationalTags.has(t)); + const meaningfulTags = (n.tags ?? []).filter((t) => !organizationalTags.has(t)); map.set(n.id, new Set(meaningfulTags)); } return map; From fce2a0fe98d849f154e8ff5a522b4aae6634b3bf Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Sun, 19 Jul 2026 00:26:55 +0530 Subject: [PATCH 20/28] ANG-008:few bug fixes --- src/data/Database/VectorDatabase.ts | 13 ++++++++++--- .../JoplinNativeProvider.test.ts | 0 .../JoplinNativeProvider.ts | 0 src/services/similarity/SimilarityEngine.ts | 14 ++++++++------ 4 files changed, 18 insertions(+), 9 deletions(-) rename src/services/embeddings/{Providers => providers}/JoplinNativeProvider.test.ts (100%) rename src/services/embeddings/{Providers => providers}/JoplinNativeProvider.ts (100%) diff --git a/src/data/Database/VectorDatabase.ts b/src/data/Database/VectorDatabase.ts index 598bc31..a3a79b7 100644 --- a/src/data/Database/VectorDatabase.ts +++ b/src/data/Database/VectorDatabase.ts @@ -13,6 +13,7 @@ interface Sqlite3Database { params: unknown[], callback: (err: Error | null, rows: unknown[]) => void ): void; + close(callback?: (err: Error | null) => void): void; } /** @@ -37,15 +38,21 @@ export class VectorDatabase implements IVectorDatabase { /** * Opens (creating if needed) the vector cache database. Safe to call - * repeatedly. A failed open is not cached: `opening` is reset on - * rejection so a later call can retry (e.g. after a transient lock), - * instead of every future open() re-awaiting the same stale rejection. + * repeatedly. A failed open is not cached: both `opening` and `db` are + * reset on rejection so a later call can retry from scratch, instead of + * either re-awaiting the same stale rejection or (if the connection + * itself succeeded but schema creation failed) treating a half-open + * database as ready forever. */ public async open(): Promise { if (this.db) return; if (!this.opening) { this.opening = this.openInternal().catch((e) => { this.opening = null; + if (this.db) { + this.db.close(); + this.db = null; + } throw e; }); } diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.test.ts b/src/services/embeddings/providers/JoplinNativeProvider.test.ts similarity index 100% rename from src/services/embeddings/Providers/JoplinNativeProvider.test.ts rename to src/services/embeddings/providers/JoplinNativeProvider.test.ts diff --git a/src/services/embeddings/Providers/JoplinNativeProvider.ts b/src/services/embeddings/providers/JoplinNativeProvider.ts similarity index 100% rename from src/services/embeddings/Providers/JoplinNativeProvider.ts rename to src/services/embeddings/providers/JoplinNativeProvider.ts diff --git a/src/services/similarity/SimilarityEngine.ts b/src/services/similarity/SimilarityEngine.ts index f9ff86b..3ec2ada 100644 --- a/src/services/similarity/SimilarityEngine.ts +++ b/src/services/similarity/SimilarityEngine.ts @@ -130,8 +130,7 @@ export class SimilarityEngine { return this.computeCosinePairs(); } - const seen = new Set(); - const pairs: SimilarityPair[] = []; + const pairs = new Map(); let successCount = 0; let firstError: unknown = null; @@ -149,13 +148,16 @@ export class SimilarityEngine { } const key = this.makePairKey(noteId, r.noteId); - if (seen.has(key)) continue; - seen.add(key); + const existing = pairs.get(key); + if (existing) { + existing.score = Math.max(existing.score, r.score); + continue; + } const [source, target] = noteId < r.noteId ? [noteId, r.noteId] : [r.noteId, noteId]; - pairs.push({ source, target, score: r.score }); + pairs.set(key, { source, target, score: r.score }); } } catch (e) { if (firstError === null) { @@ -177,7 +179,7 @@ export class SimilarityEngine { return this.computeCosinePairs(); } - return pairs; + return Array.from(pairs.values()); } /** Dot product of two same-length vectors. */ From d73de8067f7f6e2a9cc61b0761e21f484813b1a0 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Tue, 21 Jul 2026 18:29:54 +0530 Subject: [PATCH 21/28] =?UTF-8?q?ANG-009:=20=E2=80=A8Wire=20embedding=20pi?= =?UTF-8?q?peline=20into=20the=20graph=20command?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- jest.config.js | 1 + src/index.ts | 62 +++++- src/services/AnalysisController.test.ts | 178 ++++++++++++++++++ src/services/AnalysisController.ts | 99 ++++++++++ src/services/graph/GraphBuilder.test.ts | 14 ++ src/services/graph/GraphBuilder.ts | 6 +- src/services/settings/GraphSettings.test.ts | 82 ++++++++ src/services/settings/GraphSettings.ts | 69 +++++++ .../similarity/SimilarityEngine.test.ts | 51 +++++ src/services/similarity/SimilarityEngine.ts | 9 +- src/tests/mocks/joplin.ts | 10 + src/ui/App.ts | 2 + src/ui/components/AnalysisProgress.ts | 12 ++ src/ui/components/Header.ts | 4 - src/ui/graph-view.js | 30 +++ src/ui/styles/panel.css | 45 ++++- src/ui/webview.ts | 12 ++ 17 files changed, 664 insertions(+), 22 deletions(-) create mode 100644 src/services/AnalysisController.test.ts create mode 100644 src/services/AnalysisController.ts create mode 100644 src/services/settings/GraphSettings.test.ts create mode 100644 src/services/settings/GraphSettings.ts create mode 100644 src/ui/components/AnalysisProgress.ts diff --git a/jest.config.js b/jest.config.js index 6912726..7fb9f73 100644 --- a/jest.config.js +++ b/jest.config.js @@ -11,6 +11,7 @@ module.exports = { moduleNameMapper: { '^api$': '/src/tests/mocks/joplin.ts', + '^api/types$': '/api/types.ts', }, clearMocks: true, diff --git a/src/index.ts b/src/index.ts index af8d9da..7669b89 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,13 +1,28 @@ import joplin from 'api'; import { MenuItemLocation } from 'api/types'; -import { initializeAiNoteGraphPanel, showAiNoteGraphPanel, postGraphData } from './ui/webview'; +import { + initializeAiNoteGraphPanel, + showAiNoteGraphPanel, + postGraphData, + postStatus, + postProgress, +} from './ui/webview'; import { NoteRepository } from './data/NoteRepository'; import { NotePreprocessor } from './data/NotePreprocessor'; import { Note } from './data/Types'; -import { GraphBuilder } from './services/graph/GraphBuilder'; +import { AnalysisController } from './services/AnalysisController'; +import { registerGraphSettings, isAiAnalysisEnabled } from './services/settings/GraphSettings'; const SHOW_NOTE_GRAPH_COMMAND = 'showNoteGraph'; const SHOW_NOTE_GRAPH_MENU_ITEM = 'showNoteGraphMenuItem'; +const NOTE_GRAPH_SETTING_KEYS = [ + 'noteGraph.aiAnalysisEnabled', + 'noteGraph.similarityThreshold', + 'noteGraph.maxEdgesPerNote', +]; + +const analysisController = new AnalysisController(); +let lastLoadedNotes: Note[] | null = null; /** * Loads all notes from the Joplin API and enriches them with links and tags. @@ -22,6 +37,18 @@ export const loadNotes = async (): Promise => { return enrichedNotes; }; +/** Embeds notes (if AI analysis is on and ready) and pushes whichever graph results. */ +const runSemanticAnalysis = async (notes: Note[]): Promise => { + const { graphData, usedAi } = await analysisController.embedAndBuildSemantic(notes, (progress) => { + void postProgress(progress.current, progress.total); + }); + + await postGraphData(graphData); + if (!usedAi && (await isAiAnalysisEnabled())) { + await postStatus('AI analysis unavailable - showing structural graph.'); + } +}; + const noteGraphCommand = { name: SHOW_NOTE_GRAPH_COMMAND, label: 'Show Note Graph', @@ -29,16 +56,39 @@ const noteGraphCommand = { try { const enrichedNotes = await loadNotes(); console.info(`Loaded ${enrichedNotes.length} notes.`); - const builder = new GraphBuilder(); - const graphData = builder.build(enrichedNotes); - await postGraphData(graphData); + lastLoadedNotes = enrichedNotes; + + await postGraphData(analysisController.buildStructural(enrichedNotes)); await showAiNoteGraphPanel(); + + await runSemanticAnalysis(enrichedNotes); } catch (error) { console.error('Failed to load note graph:', error); } }, }; +/** + * Reacts to changes made in Tools → Options → Note Graph. Toggling AI analysis + * re-runs the full analysis; changing threshold/top-K only recomputes from the + * already-embedded vectors. No-ops if the graph hasn't been opened yet. + */ +const handleSettingsChange = async (event: { keys: string[] }): Promise => { + if (!lastLoadedNotes || !event.keys.some((key) => NOTE_GRAPH_SETTING_KEYS.includes(key))) { + return; + } + + if (event.keys.includes('noteGraph.aiAnalysisEnabled')) { + await runSemanticAnalysis(lastLoadedNotes); + return; + } + + const graphData = await analysisController.recompute(); + if (graphData) { + await postGraphData(graphData); + } +}; + const registerCommands = async (): Promise => { await joplin.commands.register(noteGraphCommand); }; @@ -54,6 +104,8 @@ const registerMenuItems = async (): Promise => { joplin.plugins.register({ onStart: async function () { console.info('Note Graph plugin started.'); + await registerGraphSettings(); + await joplin.settings.onChange(handleSettingsChange); await initializeAiNoteGraphPanel(); await registerCommands(); await registerMenuItems(); diff --git a/src/services/AnalysisController.test.ts b/src/services/AnalysisController.test.ts new file mode 100644 index 0000000..b7b928c --- /dev/null +++ b/src/services/AnalysisController.test.ts @@ -0,0 +1,178 @@ +import { AnalysisController } from './AnalysisController'; +import { GraphBuilder } from './graph/GraphBuilder'; +import { ProviderResolver } from './embeddings/ProviderResolver'; +import { EmbeddingOrchestrator } from './embeddings/Orchestrator'; +import { isAiAnalysisEnabled, getSimilaritySettings } from './settings/GraphSettings'; +import { Note } from '../data/Types'; +import { EmbeddingProvider } from './embeddings/Types'; + +jest.mock('./graph/GraphBuilder'); +jest.mock('./embeddings/ProviderResolver'); +jest.mock('./embeddings/Orchestrator'); +jest.mock('./settings/GraphSettings'); +jest.mock('../data/Database/VectorRepository', () => ({ + VectorRepository: jest.fn(), +})); + +const MockGraphBuilder = GraphBuilder as jest.MockedClass; +const MockProviderResolver = ProviderResolver as jest.Mocked; +const MockOrchestrator = EmbeddingOrchestrator as jest.MockedClass; +const mockIsAiAnalysisEnabled = isAiAnalysisEnabled as jest.Mock; +const mockGetSimilaritySettings = getSimilaritySettings as jest.Mock; + +function note(id: string): Note { + return { + id, + parent_id: 'p1', + title: id, + body: '', + created_time: 0, + updated_time: 1, + links: [], + tags: [], + }; +} + +const fakeProvider: EmbeddingProvider = { + id: 'joplin-native', + modelName: 'test-model', + fetchVectorsByNoteIds: jest.fn(), +}; + +describe('AnalysisController', () => { + let mockBuilder: jest.Mocked; + let controller: AnalysisController; + let mockOrchestratorInstance: { + setProvider: jest.Mock; + setCache: jest.Mock; + setOnProgress: jest.Mock; + embedNotes: jest.Mock; + }; + + beforeEach(() => { + jest.clearAllMocks(); + mockBuilder = new MockGraphBuilder() as jest.Mocked; + mockBuilder.build.mockReturnValue({ nodes: [], edges: [] }); + mockBuilder.buildWithSimilarity.mockResolvedValue({ nodes: [], edges: [] }); + mockGetSimilaritySettings.mockResolvedValue({ threshold: 0.5, topK: 5 }); + controller = new AnalysisController(mockBuilder); + + mockOrchestratorInstance = { + setProvider: jest.fn(), + setCache: jest.fn(), + setOnProgress: jest.fn(), + embedNotes: jest.fn().mockResolvedValue({ embeddedNotes: [], errors: [] }), + }; + MockOrchestrator.mockImplementation( + () => mockOrchestratorInstance as unknown as EmbeddingOrchestrator + ); + }); + + describe('buildStructural', () => { + it('delegates directly to GraphBuilder.build', () => { + const notes = [note('a')]; + controller.buildStructural(notes); + expect(mockBuilder.build).toHaveBeenCalledWith(notes); + }); + }); + + describe('embedAndBuildSemantic', () => { + it('falls back to the structural graph when the setting is off', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(false); + const notes = [note('a')]; + + const result = await controller.embedAndBuildSemantic(notes); + + expect(result.usedAi).toBe(false); + expect(mockBuilder.build).toHaveBeenCalledWith(notes); + expect(MockProviderResolver.resolveWithValidation).not.toHaveBeenCalled(); + }); + + it('falls back to the structural graph when provider resolution throws', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockRejectedValue( + new Error('joplin.ai is not available') + ); + const notes = [note('a')]; + + const result = await controller.embedAndBuildSemantic(notes); + + expect(result.usedAi).toBe(false); + expect(mockBuilder.build).toHaveBeenCalledWith(notes); + expect(mockBuilder.buildWithSimilarity).not.toHaveBeenCalled(); + }); + + it('falls back to the structural graph when embedding produces no vectors', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + mockOrchestratorInstance.embedNotes.mockResolvedValue({ + embeddedNotes: [], + errors: [{ noteId: 'a', error: 'Note not yet indexed by Joplin AI.' }], + }); + const notes = [note('a')]; + + const result = await controller.embedAndBuildSemantic(notes); + + expect(result.usedAi).toBe(false); + expect(mockBuilder.buildWithSimilarity).not.toHaveBeenCalled(); + }); + + it('builds the semantic graph on success', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + const embeddedNotes = [{ note: note('a'), embedding: [1, 0] }]; + mockOrchestratorInstance.embedNotes.mockResolvedValue({ embeddedNotes, errors: [] }); + const notes = [note('a')]; + + const result = await controller.embedAndBuildSemantic(notes); + + expect(result.usedAi).toBe(true); + expect(mockOrchestratorInstance.setProvider).toHaveBeenCalledWith(fakeProvider); + expect(mockOrchestratorInstance.setCache).toHaveBeenCalled(); + expect(mockBuilder.buildWithSimilarity).toHaveBeenCalledWith(notes, embeddedNotes, 0.5, 5); + }); + + it('wires an onProgress callback into the orchestrator when provided', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + mockOrchestratorInstance.embedNotes.mockResolvedValue({ + embeddedNotes: [{ note: note('a'), embedding: [1, 0] }], + errors: [], + }); + const onProgress = jest.fn(); + + await controller.embedAndBuildSemantic([note('a')], onProgress); + + expect(mockOrchestratorInstance.setOnProgress).toHaveBeenCalledWith(onProgress); + }); + }); + + describe('recompute', () => { + it('returns null when nothing has been embedded yet', async () => { + const result = await controller.recompute(); + expect(result).toBeNull(); + expect(mockBuilder.buildWithSimilarity).not.toHaveBeenCalled(); + }); + + it('reuses the last embedded notes without re-embedding, using the current settings', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + const notes = [note('a')]; + const embeddedNotes = [{ note: note('a'), embedding: [1, 0] }]; + mockOrchestratorInstance.embedNotes.mockResolvedValue({ embeddedNotes, errors: [] }); + await controller.embedAndBuildSemantic(notes); + + jest.clearAllMocks(); + mockGetSimilaritySettings.mockResolvedValue({ threshold: 0.7, topK: 3 }); + await controller.recompute(); + + expect(mockOrchestratorInstance.embedNotes).not.toHaveBeenCalled(); + expect(mockBuilder.buildWithSimilarity).toHaveBeenCalledWith( + notes, + embeddedNotes, + 0.7, + 3 + ); + }); + }); +}); diff --git a/src/services/AnalysisController.ts b/src/services/AnalysisController.ts new file mode 100644 index 0000000..4f7df35 --- /dev/null +++ b/src/services/AnalysisController.ts @@ -0,0 +1,99 @@ +import { Note } from '../data/Types'; +import { GraphBuilder } from './graph/GraphBuilder'; +import { GraphData } from './graph/types'; +import { VectorRepository } from '../data/Database/VectorRepository'; +import { ProviderResolver } from './embeddings/ProviderResolver'; +import { EmbeddingOrchestrator } from './embeddings/Orchestrator'; +import { EmbeddedNote, EmbeddingProvider, BatchProgress } from './embeddings/Types'; +import { isAiAnalysisEnabled, getSimilaritySettings } from './settings/GraphSettings'; + +export interface SemanticBuildResult { + graphData: GraphData; + usedAi: boolean; +} + +/** + * Coordinates turning notes into a GraphData, deciding between the plain + * structural graph and the AI-enhanced one, and caching the last successful + * embedding so threshold/top-K changes can recompute without re-embedding. + */ +export class AnalysisController { + private lastNotes: Note[] | null = null; + private lastEmbeddedNotes: EmbeddedNote[] | null = null; + + public constructor(private readonly builder = new GraphBuilder()) {} + + public buildStructural(notes: Note[]): GraphData { + return this.builder.build(notes); + } + + /** + * `usedAi: false` covers two different situations the caller must treat the + * same way (render the structural graph) but may want to message + * differently: AI analysis is off, or it's on but unavailable/failed. Check + * `isAiAnalysisEnabled()` separately if that distinction matters. + */ + public async embedAndBuildSemantic( + notes: Note[], + onProgress?: (progress: BatchProgress) => void + ): Promise { + const embeddedNotes = await this.tryEmbed(notes, onProgress); + + if (!embeddedNotes) { + return { graphData: this.builder.build(notes), usedAi: false }; + } + + this.lastNotes = notes; + this.lastEmbeddedNotes = embeddedNotes; + + console.info(`AI analysis: ${embeddedNotes.length}/${notes.length} notes embedded, building semantic graph.`); + const { threshold, topK } = await getSimilaritySettings(); + const graphData = await this.builder.buildWithSimilarity(notes, embeddedNotes, threshold, topK); + return { graphData, usedAi: true }; + } + + /** Rebuilds the graph from the last successful embedding using the current threshold/top-K settings. */ + public async recompute(): Promise { + if (!this.lastNotes || !this.lastEmbeddedNotes) { + return null; + } + const { threshold, topK } = await getSimilaritySettings(); + console.info( + `Recomputing graph: threshold=${threshold}, topK=${topK}, ${this.lastEmbeddedNotes.length} cached vectors.` + ); + return this.builder.buildWithSimilarity(this.lastNotes, this.lastEmbeddedNotes, threshold, topK); + } + + /** Returns null on any failure (setting off, provider unavailable, nothing embedded) — never throws, so the caller can always fall back to the structural graph. */ + private async tryEmbed( + notes: Note[], + onProgress?: (progress: BatchProgress) => void + ): Promise { + if (!(await isAiAnalysisEnabled())) { + return null; + } + + let provider: EmbeddingProvider; + try { + provider = await ProviderResolver.resolveWithValidation(); + } catch (e) { + console.error('AI analysis unavailable, falling back to structural graph:', e); + return null; + } + + const orchestrator = new EmbeddingOrchestrator(); + orchestrator.setProvider(provider); + orchestrator.setCache(new VectorRepository()); + if (onProgress) { + orchestrator.setOnProgress(onProgress); + } + + const { embeddedNotes, errors } = await orchestrator.embedNotes(notes); + if (embeddedNotes.length === 0) { + console.error('AI analysis produced no embeddings, falling back to structural graph:', errors); + return null; + } + + return embeddedNotes; + } +} diff --git a/src/services/graph/GraphBuilder.test.ts b/src/services/graph/GraphBuilder.test.ts index 1d4c721..06af347 100644 --- a/src/services/graph/GraphBuilder.test.ts +++ b/src/services/graph/GraphBuilder.test.ts @@ -128,5 +128,19 @@ describe('GraphBuilder', () => { expect(result.nodes).toHaveLength(2); expect(result.edges).toEqual([]); }); + + it('forwards a custom threshold and top-K to SimilarityEngine.compute', async () => { + mockEdgeFactory.createEdges.mockReturnValue([]); + mockEdgeFactory.createSemanticEdges.mockReturnValue([]); + const computeMock = jest.fn().mockResolvedValue([]); + MockSimilarityEngine.mockImplementation( + () => ({ compute: computeMock } as unknown as SimilarityEngine) + ); + + const notes = [note('a', 'A'), note('b', 'B')]; + await builder.buildWithSimilarity(notes, [], 0.7, 3); + + expect(computeMock).toHaveBeenCalledWith(0.7, 3); + }); }); }); diff --git a/src/services/graph/GraphBuilder.ts b/src/services/graph/GraphBuilder.ts index 85aa096..2654fc6 100644 --- a/src/services/graph/GraphBuilder.ts +++ b/src/services/graph/GraphBuilder.ts @@ -27,12 +27,14 @@ export class GraphBuilder { */ public async buildWithSimilarity( notes: Note[], - embeddedNotes: EmbeddedNote[] + embeddedNotes: EmbeddedNote[], + threshold?: number, + topK?: number ): Promise { const structuralEdges = this.edgeFactory.createEdges(notes); const engine = new SimilarityEngine(notes, embeddedNotes); - const pairs = await engine.compute(); + const pairs = await engine.compute(threshold, topK); const semanticEdges = this.edgeFactory.createSemanticEdges(pairs); const allEdges = [...structuralEdges, ...semanticEdges]; diff --git a/src/services/settings/GraphSettings.test.ts b/src/services/settings/GraphSettings.test.ts new file mode 100644 index 0000000..05e2287 --- /dev/null +++ b/src/services/settings/GraphSettings.test.ts @@ -0,0 +1,82 @@ +import joplin from 'api'; +import { SettingItemType } from 'api/types'; +import { registerGraphSettings, isAiAnalysisEnabled, getSimilaritySettings } from './GraphSettings'; + +describe('GraphSettings', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('registerGraphSettings', () => { + it('registers a section and all note-graph settings', async () => { + await registerGraphSettings(); + + expect(joplin.settings.registerSection).toHaveBeenCalledWith( + 'noteGraph', + expect.objectContaining({ label: expect.any(String) }) + ); + expect(joplin.settings.registerSettings).toHaveBeenCalledWith( + expect.objectContaining({ + 'noteGraph.aiAnalysisEnabled': expect.objectContaining({ + type: SettingItemType.Bool, + value: false, + public: true, + section: 'noteGraph', + }), + 'noteGraph.similarityThreshold': expect.objectContaining({ + type: SettingItemType.Int, + value: 50, + minimum: 0, + maximum: 100, + public: true, + section: 'noteGraph', + }), + 'noteGraph.maxEdgesPerNote': expect.objectContaining({ + type: SettingItemType.Int, + value: 5, + minimum: 1, + maximum: 20, + public: true, + section: 'noteGraph', + }), + }) + ); + }); + }); + + describe('isAiAnalysisEnabled', () => { + it('reads the aiAnalysisEnabled key', async () => { + (joplin.settings.value as jest.Mock).mockResolvedValue(true); + + const result = await isAiAnalysisEnabled(); + + expect(joplin.settings.value).toHaveBeenCalledWith('noteGraph.aiAnalysisEnabled'); + expect(result).toBe(true); + }); + + it('returns false when the setting is false', async () => { + (joplin.settings.value as jest.Mock).mockResolvedValue(false); + + const result = await isAiAnalysisEnabled(); + + expect(result).toBe(false); + }); + }); + + describe('getSimilaritySettings', () => { + it('reads both keys and converts threshold from a 0-100 percentage to a 0-1 fraction', async () => { + (joplin.settings.values as jest.Mock).mockResolvedValue({ + 'noteGraph.similarityThreshold': 70, + 'noteGraph.maxEdgesPerNote': 8, + }); + + const result = await getSimilaritySettings(); + + expect(joplin.settings.values).toHaveBeenCalledWith([ + 'noteGraph.similarityThreshold', + 'noteGraph.maxEdgesPerNote', + ]); + expect(result).toEqual({ threshold: 0.7, topK: 8 }); + }); + }); +}); diff --git a/src/services/settings/GraphSettings.ts b/src/services/settings/GraphSettings.ts new file mode 100644 index 0000000..5892a20 --- /dev/null +++ b/src/services/settings/GraphSettings.ts @@ -0,0 +1,69 @@ +import joplin from 'api'; +import { SettingItemType } from 'api/types'; +import { DEFAULT_THRESHOLD, TOP_K } from '../similarity/ThresholdPresets'; + +const SECTION_NAME = 'noteGraph'; +const AI_ANALYSIS_ENABLED_KEY = 'noteGraph.aiAnalysisEnabled'; +const SIMILARITY_THRESHOLD_KEY = 'noteGraph.similarityThreshold'; +const MAX_EDGES_PER_NOTE_KEY = 'noteGraph.maxEdgesPerNote'; + +/** + * Registers plugin settings. Registration is dynamic (lost on restart), so + * this must run on every onStart — the stored value itself persists. + */ +export async function registerGraphSettings(): Promise { + await joplin.settings.registerSection(SECTION_NAME, { + label: 'Note Graph', + }); + + await joplin.settings.registerSettings({ + [AI_ANALYSIS_ENABLED_KEY]: { + value: false, + type: SettingItemType.Bool, + public: true, + section: SECTION_NAME, + label: 'Enable AI-based semantic analysis', + description: + 'Adds semantic similarity edges to the note graph using Joplin AI. Requires Joplin AI to be enabled with a ready embedding index (Settings → AI).', + }, + [SIMILARITY_THRESHOLD_KEY]: { + value: Math.round(DEFAULT_THRESHOLD * 100), + type: SettingItemType.Int, + minimum: 0, + maximum: 100, + step: 5, + public: true, + section: SECTION_NAME, + label: 'Similarity threshold (%)', + description: 'Lower value = more semantic edges. Only applies when AI analysis is enabled.', + }, + [MAX_EDGES_PER_NOTE_KEY]: { + value: TOP_K, + type: SettingItemType.Int, + minimum: 1, + maximum: 20, + step: 1, + public: true, + section: SECTION_NAME, + label: 'Max semantic edges per note (top-K)', + description: 'Only applies when AI analysis is enabled.', + }, + }); +} + +export async function isAiAnalysisEnabled(): Promise { + return await joplin.settings.value(AI_ANALYSIS_ENABLED_KEY); +} + +/** + * Joplin settings have no float/slider type, only Int — the threshold is + * stored as a 0-100 percentage and converted here to the 0-1 scale + * SimilarityEngine expects. + */ +export async function getSimilaritySettings(): Promise<{ threshold: number; topK: number }> { + const values = await joplin.settings.values([SIMILARITY_THRESHOLD_KEY, MAX_EDGES_PER_NOTE_KEY]); + return { + threshold: values[SIMILARITY_THRESHOLD_KEY] / 100, + topK: values[MAX_EDGES_PER_NOTE_KEY], + }; +} diff --git a/src/services/similarity/SimilarityEngine.test.ts b/src/services/similarity/SimilarityEngine.test.ts index a94d374..917e214 100644 --- a/src/services/similarity/SimilarityEngine.test.ts +++ b/src/services/similarity/SimilarityEngine.test.ts @@ -468,6 +468,57 @@ describe('SimilarityEngine', () => { }); }); + describe('custom threshold and top-K overrides', () => { + it('applies a stricter caller-supplied threshold instead of DEFAULT_THRESHOLD', async () => { + const notes = [makeNote('a', 'A'), makeNote('b', 'B')]; + const embedded = [embed('a', [1, 0]), embed('b', [0.95, 0.3])]; + + const engine = new SimilarityEngine(notes, embedded); + const defaultPairs = await engine.compute(); + const strictPairs = await engine.compute(1.2); + + expect(defaultPairs).toHaveLength(1); + expect(strictPairs).toEqual([]); + }); + + it('applies a looser caller-supplied threshold that admits a pair DEFAULT_THRESHOLD would reject', async () => { + // Raw cosine 0.35 (above SEMANTIC_FLOOR) plus the same-day temporal + // bonus (0.1) lands at 0.45 — below DEFAULT_THRESHOLD (0.5) but above + // a caller-supplied 0.4. + const notes = [makeNote('a', 'A'), makeNote('b', 'B')]; + const embedded = [embed('a', [1, 0]), embed('b', [0.35, Math.sqrt(1 - 0.35 * 0.35)])]; + + const engine = new SimilarityEngine(notes, embedded); + const defaultPairs = await engine.compute(); + const loosePairs = await engine.compute(0.4); + + expect(defaultPairs).toEqual([]); + expect(loosePairs).toHaveLength(1); + }); + + it('applies a caller-supplied top-K instead of TOP_K', async () => { + // selectTopK is a per-note union (a pair survives if *either* endpoint + // keeps it in its own top-K), so topK=0 is the only value that + // unambiguously proves the override took effect: every note's own + // kept list is empty, so no pair can survive from any side. + const notes = []; + const embedded = []; + for (let i = 0; i < 6; i++) { + notes.push(makeNote(`n${i}`, `Note ${i}`)); + embedded.push( + embed(`n${i}`, [Math.cos((i * Math.PI) / 3), Math.sin((i * Math.PI) / 3)]) + ); + } + + const engine = new SimilarityEngine(notes, embedded); + const defaultPairs = await engine.compute(); + const zeroKPairs = await engine.compute(undefined, 0); + + expect(defaultPairs.length).toBeGreaterThan(0); + expect(zeroKPairs).toEqual([]); + }); + }); + describe('SEMANTIC_FLOOR and threshold ordering', () => { it('rejects a below-floor pair even with a shared tag', async () => { // a and b are nearly orthogonal (cosine ~0), share a tag but are not diff --git a/src/services/similarity/SimilarityEngine.ts b/src/services/similarity/SimilarityEngine.ts index 3ec2ada..b879152 100644 --- a/src/services/similarity/SimilarityEngine.ts +++ b/src/services/similarity/SimilarityEngine.ts @@ -50,7 +50,10 @@ export class SimilarityEngine { * absolute meaning) is what actually guarantees that tags alone can never * manufacture an edge out of a weak semantic score. */ - public async compute(): Promise { + public async compute( + threshold: number = DEFAULT_THRESHOLD, + topK: number = TOP_K + ): Promise { if (this.noteIds.length <= 1) { return []; } @@ -69,8 +72,8 @@ export class SimilarityEngine { const normalized = this.normalize(aboveFloor); const enriched = this.addBonusPoints(normalized); - const aboveThreshold = this.filterBelowThreshold(enriched, DEFAULT_THRESHOLD); - const topPairs = this.selectTopK(aboveThreshold, TOP_K); + const aboveThreshold = this.filterBelowThreshold(enriched, threshold); + const topPairs = this.selectTopK(aboveThreshold, topK); return topPairs; } diff --git a/src/tests/mocks/joplin.ts b/src/tests/mocks/joplin.ts index 0b6e5d4..a505703 100644 --- a/src/tests/mocks/joplin.ts +++ b/src/tests/mocks/joplin.ts @@ -5,11 +5,21 @@ const joplinAi = { chat: jest.fn(), }; +const joplinSettings = { + registerSection: jest.fn(), + registerSettings: jest.fn(), + value: jest.fn(), + values: jest.fn(), + setValue: jest.fn(), + onChange: jest.fn(), +}; + const joplin = { data: { get: jest.fn(), }, ai: joplinAi, + settings: joplinSettings, }; export default joplin; diff --git a/src/ui/App.ts b/src/ui/App.ts index 9f0c3fa..42865a1 100644 --- a/src/ui/App.ts +++ b/src/ui/App.ts @@ -2,6 +2,7 @@ import { renderHeader } from './components/Header'; import { renderLegend } from './components/Legend'; import { renderStatsBar } from './components/StatsBar'; import { renderGraphControls } from './components/GraphControls'; +import { renderAnalysisProgress } from './components/AnalysisProgress'; const renderPanelHtml = (): string => { return ` @@ -9,6 +10,7 @@ const renderPanelHtml = (): string => { ${renderHeader()} ${renderLegend()} ${renderStatsBar()} + ${renderAnalysisProgress()}
${renderGraphControls()} Loading graph... diff --git a/src/ui/components/AnalysisProgress.ts b/src/ui/components/AnalysisProgress.ts new file mode 100644 index 0000000..3fde049 --- /dev/null +++ b/src/ui/components/AnalysisProgress.ts @@ -0,0 +1,12 @@ +const renderAnalysisProgress = (): string => { + return ` + + `; +}; + +export { renderAnalysisProgress }; diff --git a/src/ui/components/Header.ts b/src/ui/components/Header.ts index 40e64a5..2b207bc 100644 --- a/src/ui/components/Header.ts +++ b/src/ui/components/Header.ts @@ -4,8 +4,6 @@ type HeaderProps = { const LogoSvg = ``; -const SettingsSvg = ``; - const CloseSvg = ``; const renderHeader = (props: HeaderProps = {}): string => { @@ -16,8 +14,6 @@ const renderHeader = (props: HeaderProps = {}): string => { ${props.title ?? 'Note Graph'}
- -
diff --git a/src/ui/graph-view.js b/src/ui/graph-view.js index 4fa389c..de63e51 100644 --- a/src/ui/graph-view.js +++ b/src/ui/graph-view.js @@ -34,6 +34,9 @@ var statusEl; var pollTimer; var tooltipEl; var nodeStats; +var progressEl; +var progressFillEl; +var progressLabelEl; function showStatus(text) { if (statusEl) { @@ -48,6 +51,21 @@ function hideStatus() { } } +/** Updates the progress bar below the stats bar with an "embedding N/M notes" state. */ +function showProgress(current, total) { + if (!progressEl || !progressFillEl || !progressLabelEl) return; + progressEl.style.display = ''; + var pct = total > 0 ? Math.round((current / total) * 100) : 0; + progressFillEl.style.width = pct + '%'; + progressLabelEl.textContent = 'Embedding notes: ' + current + '/' + total; +} + +function hideProgress() { + if (progressEl) { + progressEl.style.display = 'none'; + } +} + /** Detect whether the current Joplin theme is dark by computing luminance of --joplin-background-color. */ function isDarkTheme() { var bg = getComputedStyle(document.body).getPropertyValue('--joplin-background-color').trim(); @@ -322,6 +340,10 @@ function init() { statusEl.style.display = ''; } + progressEl = document.getElementById('analysis-progress'); + progressFillEl = document.getElementById('analysis-progress-fill'); + progressLabelEl = document.getElementById('analysis-progress-label'); + tooltipEl = document.createElement('div'); tooltipEl.className = 'graph-tooltip'; document.body.appendChild(tooltipEl); @@ -516,11 +538,19 @@ function init() { clearInterval(pollTimer); pollTimer = null; } + hideProgress(); renderGraph(message); } if (message && message.type === 'fit-to-screen') { cy.fit(undefined, 30); } + if (message && message.type === 'status' && message.text) { + hideProgress(); + showStatus(message.text); + } + if (message && message.type === 'progress') { + showProgress(message.current, message.total); + } }); } } catch (e) { diff --git a/src/ui/styles/panel.css b/src/ui/styles/panel.css index 89cc1c3..0fc6d73 100644 --- a/src/ui/styles/panel.css +++ b/src/ui/styles/panel.css @@ -58,14 +58,6 @@ body { margin-left: auto; } -.panel-header__action-sep { - width: 1px; - height: 14px; - background: rgba(128, 128, 128, 0.25); - flex-shrink: 0; - margin: 0 3px; -} - .panel-header__icon-btn { background-color: transparent; border: none; @@ -322,6 +314,43 @@ body { flex-shrink: 0; } +/* Analysis progress bar */ + +.analysis-progress { + display: flex; + align-items: center; + gap: 10px; + padding: 6px 16px; + width: 100%; + box-sizing: border-box; + flex-shrink: 0; + background: rgba(91, 155, 213, 0.07); + border-bottom: 1px solid rgba(128, 128, 128, 0.10); + font-size: 11px; + color: var(--joplin-color-faded, #888); +} + +.analysis-progress__track { + flex: 1 1 auto; + height: 5px; + border-radius: 3px; + background: rgba(128, 128, 128, 0.2); + overflow: hidden; +} + +.analysis-progress__fill { + height: 100%; + width: 0%; + background: #5b9bd5; + border-radius: 3px; + transition: width 0.2s ease-out; +} + +.analysis-progress__label { + flex-shrink: 0; + font-variant-numeric: tabular-nums; +} + /* Graph container */ #graph-container { diff --git a/src/ui/webview.ts b/src/ui/webview.ts index 25b72a3..f94c317 100644 --- a/src/ui/webview.ts +++ b/src/ui/webview.ts @@ -84,3 +84,15 @@ export const postGraphData = async (graphData: GraphData): Promise => { joplin.views.panels.postMessage(handle, { type: 'graph-data', ...graphData }); } }; + +/** Pushes a one-line status message to the panel (e.g. a fallback notice). */ +export const postStatus = async (text: string): Promise => { + const handle = getPanel(); + await joplin.views.panels.postMessage(handle, { type: 'status', text }); +}; + +/** Pushes embedding progress to the panel's progress bar. */ +export const postProgress = async (current: number, total: number): Promise => { + const handle = getPanel(); + await joplin.views.panels.postMessage(handle, { type: 'progress', current, total }); +}; From 65cf9fd97dd0bb3ee3c1d0113bb48468e80d2a15 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Thu, 23 Jul 2026 20:30:36 +0530 Subject: [PATCH 22/28] ANG-009: few bug fixes --- src/index.ts | 35 ++++++--- src/services/AnalysisController.test.ts | 100 ++++++++++++++++++++++-- src/services/AnalysisController.ts | 72 +++++++++++++---- src/services/settings/GraphSettings.ts | 9 ++- 4 files changed, 182 insertions(+), 34 deletions(-) diff --git a/src/index.ts b/src/index.ts index 7669b89..2b3d1e2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,15 +11,15 @@ import { NoteRepository } from './data/NoteRepository'; import { NotePreprocessor } from './data/NotePreprocessor'; import { Note } from './data/Types'; import { AnalysisController } from './services/AnalysisController'; -import { registerGraphSettings, isAiAnalysisEnabled } from './services/settings/GraphSettings'; +import { + registerGraphSettings, + isAiAnalysisEnabled, + AI_ANALYSIS_ENABLED_KEY, + NOTE_GRAPH_SETTING_KEYS, +} from './services/settings/GraphSettings'; const SHOW_NOTE_GRAPH_COMMAND = 'showNoteGraph'; const SHOW_NOTE_GRAPH_MENU_ITEM = 'showNoteGraphMenuItem'; -const NOTE_GRAPH_SETTING_KEYS = [ - 'noteGraph.aiAnalysisEnabled', - 'noteGraph.similarityThreshold', - 'noteGraph.maxEdgesPerNote', -]; const analysisController = new AnalysisController(); let lastLoadedNotes: Note[] | null = null; @@ -37,15 +37,23 @@ export const loadNotes = async (): Promise => { return enrichedNotes; }; -/** Embeds notes (if AI analysis is on and ready) and pushes whichever graph results. */ +/** + * Embeds notes (if AI analysis is on and ready) and pushes whichever graph results. + * A `null` result means a newer call started before this one finished — its + * data is stale, so it's dropped instead of overwriting the newer graph. + */ const runSemanticAnalysis = async (notes: Note[]): Promise => { - const { graphData, usedAi } = await analysisController.embedAndBuildSemantic(notes, (progress) => { + const result = await analysisController.embedAndBuildSemantic(notes, (progress) => { void postProgress(progress.current, progress.total); }); + if (!result) { + return; + } + const { graphData, usedAi, fallbackReason } = result; await postGraphData(graphData); if (!usedAi && (await isAiAnalysisEnabled())) { - await postStatus('AI analysis unavailable - showing structural graph.'); + await postStatus(fallbackReason ?? 'AI analysis unavailable - showing structural graph.'); } }; @@ -78,11 +86,18 @@ const handleSettingsChange = async (event: { keys: string[] }): Promise => return; } - if (event.keys.includes('noteGraph.aiAnalysisEnabled')) { + if (event.keys.includes(AI_ANALYSIS_ENABLED_KEY)) { await runSemanticAnalysis(lastLoadedNotes); return; } + // Threshold / top-K only affect semantic edges, which exist only while AI + // analysis is enabled (matches the settings' own description). Skip the + // recompute when it's off so a stale embedding cache can't resurrect edges. + if (!(await isAiAnalysisEnabled())) { + return; + } + const graphData = await analysisController.recompute(); if (graphData) { await postGraphData(graphData); diff --git a/src/services/AnalysisController.test.ts b/src/services/AnalysisController.test.ts index b7b928c..2d5eee1 100644 --- a/src/services/AnalysisController.test.ts +++ b/src/services/AnalysisController.test.ts @@ -39,6 +39,20 @@ const fakeProvider: EmbeddingProvider = { fetchVectorsByNoteIds: jest.fn(), }; +type EmbedResult = { embeddedNotes: unknown[]; errors: unknown[] }; + +/** A promise plus its own resolve function, for tests that need to control exactly when a mocked async call settles. */ +function deferredEmbedResult(): { + promise: Promise; + resolve: (result: EmbedResult) => void; +} { + let resolve!: (result: EmbedResult) => void; + const promise = new Promise((res) => { + resolve = res; + }); + return { promise, resolve }; +} + describe('AnalysisController', () => { let mockBuilder: jest.Mocked; let controller: AnalysisController; @@ -83,12 +97,13 @@ describe('AnalysisController', () => { const result = await controller.embedAndBuildSemantic(notes); - expect(result.usedAi).toBe(false); + expect(result?.usedAi).toBe(false); + expect(result?.fallbackReason).toBeUndefined(); expect(mockBuilder.build).toHaveBeenCalledWith(notes); expect(MockProviderResolver.resolveWithValidation).not.toHaveBeenCalled(); }); - it('falls back to the structural graph when provider resolution throws', async () => { + it('falls back to the structural graph when provider resolution throws, carrying the reason', async () => { mockIsAiAnalysisEnabled.mockResolvedValue(true); MockProviderResolver.resolveWithValidation.mockRejectedValue( new Error('joplin.ai is not available') @@ -97,12 +112,13 @@ describe('AnalysisController', () => { const result = await controller.embedAndBuildSemantic(notes); - expect(result.usedAi).toBe(false); + expect(result?.usedAi).toBe(false); + expect(result?.fallbackReason).toBe('joplin.ai is not available'); expect(mockBuilder.build).toHaveBeenCalledWith(notes); expect(mockBuilder.buildWithSimilarity).not.toHaveBeenCalled(); }); - it('falls back to the structural graph when embedding produces no vectors', async () => { + it('falls back to the structural graph when embedding produces no vectors, carrying the reason', async () => { mockIsAiAnalysisEnabled.mockResolvedValue(true); MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); mockOrchestratorInstance.embedNotes.mockResolvedValue({ @@ -113,7 +129,8 @@ describe('AnalysisController', () => { const result = await controller.embedAndBuildSemantic(notes); - expect(result.usedAi).toBe(false); + expect(result?.usedAi).toBe(false); + expect(result?.fallbackReason).toBe('Note not yet indexed by Joplin AI.'); expect(mockBuilder.buildWithSimilarity).not.toHaveBeenCalled(); }); @@ -126,10 +143,40 @@ describe('AnalysisController', () => { const result = await controller.embedAndBuildSemantic(notes); - expect(result.usedAi).toBe(true); + expect(result?.usedAi).toBe(true); expect(mockOrchestratorInstance.setProvider).toHaveBeenCalledWith(fakeProvider); expect(mockOrchestratorInstance.setCache).toHaveBeenCalled(); - expect(mockBuilder.buildWithSimilarity).toHaveBeenCalledWith(notes, embeddedNotes, 0.5, 5); + expect(mockBuilder.buildWithSimilarity).toHaveBeenCalledWith( + notes, + embeddedNotes, + 0.5, + 5 + ); + }); + + it('discards a run that resolves after a newer run has already started', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + + const first = deferredEmbedResult(); + const second = deferredEmbedResult(); + mockOrchestratorInstance.embedNotes + .mockReturnValueOnce(first.promise) + .mockReturnValueOnce(second.promise); + + const firstCall = controller.embedAndBuildSemantic([note('a')]); + const secondCall = controller.embedAndBuildSemantic([note('b')]); + + // The newer (second) run finishes first... + second.resolve({ embeddedNotes: [{ note: note('b'), embedding: [0, 1] }], errors: [] }); + const secondResult = await secondCall; + + // ...then the stale first run finishes after it and should be discarded. + first.resolve({ embeddedNotes: [{ note: note('a'), embedding: [1, 0] }], errors: [] }); + const firstResult = await firstCall; + + expect(secondResult?.usedAi).toBe(true); + expect(firstResult).toBeNull(); }); it('wires an onProgress callback into the orchestrator when provided', async () => { @@ -143,7 +190,44 @@ describe('AnalysisController', () => { await controller.embedAndBuildSemantic([note('a')], onProgress); - expect(mockOrchestratorInstance.setOnProgress).toHaveBeenCalledWith(onProgress); + expect(mockOrchestratorInstance.setOnProgress).toHaveBeenCalledTimes(1); + const wiredProgress = mockOrchestratorInstance.setOnProgress.mock.calls[0][0]; + wiredProgress({ current: 1, total: 1 }); + expect(onProgress).toHaveBeenCalledWith({ current: 1, total: 1 }); + }); + + it('stops forwarding progress from a run once a newer run has started', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + + const first = deferredEmbedResult(); + mockOrchestratorInstance.embedNotes + .mockReturnValueOnce(first.promise) + .mockResolvedValueOnce({ + embeddedNotes: [{ note: note('b'), embedding: [0, 1] }], + errors: [], + }); + + const onProgressFirst = jest.fn(); + const firstCall = controller.embedAndBuildSemantic([note('a')], onProgressFirst); + // Let the two internal awaits (isAiAnalysisEnabled, resolveWithValidation) settle + // so the orchestrator is constructed and wired before we grab its callback. + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + // Second call has no onProgress, so this is unambiguously the first run's wired callback. + const firstRunProgress = mockOrchestratorInstance.setOnProgress.mock.calls[0][0]; + + const secondCall = controller.embedAndBuildSemantic([note('b')]); + await secondCall; + + // The stale first run reports progress after being superseded... + firstRunProgress({ current: 1, total: 1 }); + // ...it should not reach the original caller. + expect(onProgressFirst).not.toHaveBeenCalled(); + + first.resolve({ embeddedNotes: [{ note: note('a'), embedding: [1, 0] }], errors: [] }); + await firstCall; }); }); diff --git a/src/services/AnalysisController.ts b/src/services/AnalysisController.ts index 4f7df35..e8ecae1 100644 --- a/src/services/AnalysisController.ts +++ b/src/services/AnalysisController.ts @@ -10,6 +10,8 @@ import { isAiAnalysisEnabled, getSimilaritySettings } from './settings/GraphSett export interface SemanticBuildResult { graphData: GraphData; usedAi: boolean; + /** Set when `usedAi` is false because AI analysis was on but failed — the reason to surface to the user. Absent when AI analysis is simply off. */ + fallbackReason?: string; } /** @@ -20,6 +22,7 @@ export interface SemanticBuildResult { export class AnalysisController { private lastNotes: Note[] | null = null; private lastEmbeddedNotes: EmbeddedNote[] | null = null; + private runToken = 0; public constructor(private readonly builder = new GraphBuilder()) {} @@ -30,25 +33,43 @@ export class AnalysisController { /** * `usedAi: false` covers two different situations the caller must treat the * same way (render the structural graph) but may want to message - * differently: AI analysis is off, or it's on but unavailable/failed. Check - * `isAiAnalysisEnabled()` separately if that distinction matters. + * differently: AI analysis is off, or it's on but unavailable/failed (see + * `fallbackReason`). Check `isAiAnalysisEnabled()` separately if that + * distinction matters. + * + * Returns `null` if a newer call to this method started before this one + * finished — its result is stale and superseded, so the caller should + * discard it rather than pushing it to the graph. */ public async embedAndBuildSemantic( notes: Note[], onProgress?: (progress: BatchProgress) => void - ): Promise { - const embeddedNotes = await this.tryEmbed(notes, onProgress); + ): Promise { + const token = ++this.runToken; + const guardedProgress = onProgress ? this.guardStaleProgress(token, onProgress) : undefined; + const { embeddedNotes, reason } = await this.tryEmbed(notes, guardedProgress); + + if (token !== this.runToken) { + return null; + } if (!embeddedNotes) { - return { graphData: this.builder.build(notes), usedAi: false }; + return { graphData: this.builder.build(notes), usedAi: false, fallbackReason: reason }; } this.lastNotes = notes; this.lastEmbeddedNotes = embeddedNotes; - console.info(`AI analysis: ${embeddedNotes.length}/${notes.length} notes embedded, building semantic graph.`); + console.info( + `AI analysis: ${embeddedNotes.length}/${notes.length} notes embedded, building semantic graph.` + ); const { threshold, topK } = await getSimilaritySettings(); - const graphData = await this.builder.buildWithSimilarity(notes, embeddedNotes, threshold, topK); + const graphData = await this.builder.buildWithSimilarity( + notes, + embeddedNotes, + threshold, + topK + ); return { graphData, usedAi: true }; } @@ -61,24 +82,42 @@ export class AnalysisController { console.info( `Recomputing graph: threshold=${threshold}, topK=${topK}, ${this.lastEmbeddedNotes.length} cached vectors.` ); - return this.builder.buildWithSimilarity(this.lastNotes, this.lastEmbeddedNotes, threshold, topK); + return this.builder.buildWithSimilarity( + this.lastNotes, + this.lastEmbeddedNotes, + threshold, + topK + ); + } + + /** Wraps a progress callback so it stops firing once a newer run supersedes `token` — otherwise a slow, superseded run could re-show the progress bar after a newer run already hid it by posting its finished graph. */ + private guardStaleProgress( + token: number, + onProgress: (progress: BatchProgress) => void + ): (progress: BatchProgress) => void { + return (progress) => { + if (token === this.runToken) { + onProgress(progress); + } + }; } - /** Returns null on any failure (setting off, provider unavailable, nothing embedded) — never throws, so the caller can always fall back to the structural graph. */ + /** Never throws — returns `embeddedNotes: null` on any failure (setting off, provider unavailable, nothing embedded), with `reason` set to a user-facing explanation where one is available, so the caller can always fall back to the structural graph. */ private async tryEmbed( notes: Note[], onProgress?: (progress: BatchProgress) => void - ): Promise { + ): Promise<{ embeddedNotes: EmbeddedNote[] | null; reason?: string }> { if (!(await isAiAnalysisEnabled())) { - return null; + return { embeddedNotes: null }; } let provider: EmbeddingProvider; try { provider = await ProviderResolver.resolveWithValidation(); } catch (e) { + const reason = e instanceof Error ? e.message : String(e); console.error('AI analysis unavailable, falling back to structural graph:', e); - return null; + return { embeddedNotes: null, reason }; } const orchestrator = new EmbeddingOrchestrator(); @@ -90,10 +129,13 @@ export class AnalysisController { const { embeddedNotes, errors } = await orchestrator.embedNotes(notes); if (embeddedNotes.length === 0) { - console.error('AI analysis produced no embeddings, falling back to structural graph:', errors); - return null; + console.error( + 'AI analysis produced no embeddings, falling back to structural graph:', + errors + ); + return { embeddedNotes: null, reason: errors[0]?.error }; } - return embeddedNotes; + return { embeddedNotes }; } } diff --git a/src/services/settings/GraphSettings.ts b/src/services/settings/GraphSettings.ts index 5892a20..b78153d 100644 --- a/src/services/settings/GraphSettings.ts +++ b/src/services/settings/GraphSettings.ts @@ -3,10 +3,17 @@ import { SettingItemType } from 'api/types'; import { DEFAULT_THRESHOLD, TOP_K } from '../similarity/ThresholdPresets'; const SECTION_NAME = 'noteGraph'; -const AI_ANALYSIS_ENABLED_KEY = 'noteGraph.aiAnalysisEnabled'; +export const AI_ANALYSIS_ENABLED_KEY = 'noteGraph.aiAnalysisEnabled'; const SIMILARITY_THRESHOLD_KEY = 'noteGraph.similarityThreshold'; const MAX_EDGES_PER_NOTE_KEY = 'noteGraph.maxEdgesPerNote'; +/** All Note Graph setting keys — the single source of truth for anything that needs to check "did one of our settings change?" */ +export const NOTE_GRAPH_SETTING_KEYS = [ + AI_ANALYSIS_ENABLED_KEY, + SIMILARITY_THRESHOLD_KEY, + MAX_EDGES_PER_NOTE_KEY, +]; + /** * Registers plugin settings. Registration is dynamic (lost on restart), so * this must run on every onStart — the stored value itself persists. From 1af7b2688e7fcda5334cf7934c70ee2911faf50f Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Sat, 25 Jul 2026 21:39:01 +0530 Subject: [PATCH 23/28] ANG-009:Added try-catch --- src/index.ts | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/index.ts b/src/index.ts index 2b3d1e2..e799012 100644 --- a/src/index.ts +++ b/src/index.ts @@ -86,21 +86,25 @@ const handleSettingsChange = async (event: { keys: string[] }): Promise => return; } - if (event.keys.includes(AI_ANALYSIS_ENABLED_KEY)) { - await runSemanticAnalysis(lastLoadedNotes); - return; - } + try { + if (event.keys.includes(AI_ANALYSIS_ENABLED_KEY)) { + await runSemanticAnalysis(lastLoadedNotes); + return; + } - // Threshold / top-K only affect semantic edges, which exist only while AI - // analysis is enabled (matches the settings' own description). Skip the - // recompute when it's off so a stale embedding cache can't resurrect edges. - if (!(await isAiAnalysisEnabled())) { - return; - } + // Threshold / top-K only affect semantic edges, which exist only while AI + // analysis is enabled (matches the settings' own description). Skip the + // recompute when it's off so a stale embedding cache can't resurrect edges. + if (!(await isAiAnalysisEnabled())) { + return; + } - const graphData = await analysisController.recompute(); - if (graphData) { - await postGraphData(graphData); + const graphData = await analysisController.recompute(); + if (graphData) { + await postGraphData(graphData); + } + } catch (error) { + console.error('Failed to handle note graph settings change:', error); } }; From 0700944d1d9aca3580072da6020b6f7479a8a832 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Mon, 27 Jul 2026 13:09:45 +0530 Subject: [PATCH 24/28] ANG-O10:Louvain Community detection and degree/centrality scoring --- package-lock.json | 85 +++++++- package.json | 4 +- src/services/graph/CentralityScorer.test.ts | 69 +++++++ src/services/graph/CentralityScorer.ts | 43 ++++ src/services/graph/GraphBuilder.test.ts | 43 +++- src/services/graph/GraphBuilder.ts | 35 +++- src/services/graph/LouvainDetector.test.ts | 195 +++++++++++++++++ src/services/graph/LouvainDetector.ts | 218 ++++++++++++++++++++ src/services/graph/types.ts | 2 + src/ui/graph-view.js | 132 +++++++++--- tsconfig.json | 1 + 11 files changed, 786 insertions(+), 41 deletions(-) create mode 100644 src/services/graph/CentralityScorer.test.ts create mode 100644 src/services/graph/CentralityScorer.ts create mode 100644 src/services/graph/LouvainDetector.test.ts create mode 100644 src/services/graph/LouvainDetector.ts diff --git a/package-lock.json b/package-lock.json index 396cf14..4707ab5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,9 @@ "dependencies": { "cytoscape": "^3.34.0", "cytoscape-fcose": "^2.2.0", - "cytoscape-svg": "^0.4.0" + "cytoscape-svg": "^0.4.0", + "graphology": "^0.26.0", + "graphology-communities-louvain": "^2.0.2" }, "devDependencies": { "@types/jest": "^29.5.14", @@ -2440,7 +2442,6 @@ "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.8.x" @@ -2808,6 +2809,62 @@ "dev": true, "license": "ISC" }, + "node_modules/graphology": { + "version": "0.26.0", + "resolved": "https://registry.npmjs.org/graphology/-/graphology-0.26.0.tgz", + "integrity": "sha512-8SSImzgUUYC89Z042s+0r/vMibY7GX/Emz4LDO5e7jYXhuoWfHISPFJYjpRLUSJGq6UQ6xlenvX1p/hJdfXuXg==", + "license": "MIT", + "dependencies": { + "events": "^3.3.0" + }, + "peerDependencies": { + "graphology-types": ">=0.24.0" + } + }, + "node_modules/graphology-communities-louvain": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/graphology-communities-louvain/-/graphology-communities-louvain-2.0.2.tgz", + "integrity": "sha512-zt+2hHVPYxjEquyecxWXoUoIuN/UvYzsvI7boDdMNz0rRvpESQ7+e+Ejv6wK7AThycbZXuQ6DkG8NPMCq6XwoA==", + "license": "MIT", + "dependencies": { + "graphology-indices": "^0.17.0", + "graphology-utils": "^2.4.4", + "mnemonist": "^0.39.0", + "pandemonium": "^2.4.1" + }, + "peerDependencies": { + "graphology-types": ">=0.19.0" + } + }, + "node_modules/graphology-indices": { + "version": "0.17.0", + "resolved": "https://registry.npmjs.org/graphology-indices/-/graphology-indices-0.17.0.tgz", + "integrity": "sha512-A7RXuKQvdqSWOpn7ZVQo4S33O0vCfPBnUSf7FwE0zNCasqwZVUaCXePuWo5HBpWw68KJcwObZDHpFk6HKH6MYQ==", + "license": "MIT", + "dependencies": { + "graphology-utils": "^2.4.2", + "mnemonist": "^0.39.0" + }, + "peerDependencies": { + "graphology-types": ">=0.20.0" + } + }, + "node_modules/graphology-types": { + "version": "0.24.8", + "resolved": "https://registry.npmjs.org/graphology-types/-/graphology-types-0.24.8.tgz", + "integrity": "sha512-hDRKYXa8TsoZHjgEaysSRyPdT6uB78Ci8WnjgbStlQysz7xR52PInxNsmnB7IBOM1BhikxkNyCVEFgmPKnpx3Q==", + "license": "MIT", + "peer": true + }, + "node_modules/graphology-utils": { + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/graphology-utils/-/graphology-utils-2.5.2.tgz", + "integrity": "sha512-ckHg8MXrXJkOARk56ZaSCM1g1Wihe2d6iTmz1enGOz4W/l831MBCKSayeFQfowgF8wd+PQ4rlch/56Vs/VZLDQ==", + "license": "MIT", + "peerDependencies": { + "graphology-types": ">=0.23.0" + } + }, "node_modules/handlebars": { "version": "4.7.9", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.9.tgz", @@ -4331,6 +4388,15 @@ "node": ">=10" } }, + "node_modules/mnemonist": { + "version": "0.39.8", + "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.39.8.tgz", + "integrity": "sha512-vyWo2K3fjrUw8YeeZ1zF0fy6Mu59RHokURlld8ymdUPjMlD9EC9ov1/YPqTgqRvUN9nTr3Gqfz29LYAmu0PHPQ==", + "license": "MIT", + "dependencies": { + "obliterator": "^2.0.1" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -4389,6 +4455,12 @@ "node": ">=8" } }, + "node_modules/obliterator": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-2.0.5.tgz", + "integrity": "sha512-42CPE9AhahZRsMNslczq0ctAEtqk8Eka26QofnqC346BZdHDySk3LWka23LI7ULIw11NmltpiLagIq8gBozxTw==", + "license": "MIT" + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -4454,6 +4526,15 @@ "node": ">=6" } }, + "node_modules/pandemonium": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/pandemonium/-/pandemonium-2.4.1.tgz", + "integrity": "sha512-wRqjisUyiUfXowgm7MFH2rwJzKIr20rca5FsHXCMNm1W5YPP1hCtrZfgmQ62kP7OZ7Xt+cR858aB28lu5NX55g==", + "license": "MIT", + "dependencies": { + "mnemonist": "^0.39.2" + } + }, "node_modules/parse-json": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", diff --git a/package.json b/package.json index 47e88e3..d88bc1a 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,8 @@ "dependencies": { "cytoscape": "^3.34.0", "cytoscape-fcose": "^2.2.0", - "cytoscape-svg": "^0.4.0" + "cytoscape-svg": "^0.4.0", + "graphology": "^0.26.0", + "graphology-communities-louvain": "^2.0.2" } } diff --git a/src/services/graph/CentralityScorer.test.ts b/src/services/graph/CentralityScorer.test.ts new file mode 100644 index 0000000..bf65013 --- /dev/null +++ b/src/services/graph/CentralityScorer.test.ts @@ -0,0 +1,69 @@ +import { CentralityScorer } from './CentralityScorer'; + +describe('CentralityScorer', () => { + let scorer: CentralityScorer; + + beforeEach(() => { + scorer = new CentralityScorer(); + }); + + it('returns an empty map for an empty degree map', () => { + expect(scorer.score(new Map())).toEqual(new Map()); + }); + + it('gives every note the same mid-range size when all degrees are equal', () => { + const result = scorer.score( + new Map([ + ['a', 3], + ['b', 3], + ['c', 3], + ]) + ); + expect(result.get('a')).toBe(5); + expect(result.get('b')).toBe(5); + expect(result.get('c')).toBe(5); + }); + + it('scales the least connected note to 1 and the most connected to 10', () => { + const result = scorer.score( + new Map([ + ['a', 0], + ['b', 5], + ['c', 10], + ]) + ); + expect(result.get('a')).toBe(1); + expect(result.get('c')).toBe(10); + }); + + it('scales a mid-degree note between min and max on a log curve', () => { + const result = scorer.score( + new Map([ + ['a', 0], + ['b', 5], + ['c', 10], + ]) + ); + expect(result.get('b')).toBe(8); + }); + + it('spreads a right-skewed degree distribution instead of pinning most notes near the minimum', () => { + // A realistic shape: most notes are low (3-8), one hub is far above them. + const result = scorer.score( + new Map([ + ['a', 0], + ['b', 3], + ['c', 4], + ['d', 5], + ['e', 8], + ['hub', 24], + ]) + ); + expect(result.get('a')).toBe(1); + expect(result.get('hub')).toBe(10); + // Plain min-max would put all of these at size 1-2. The log curve + // should spread them further up the scale. + expect(result.get('b')).toBeGreaterThanOrEqual(4); + expect(result.get('e')).toBeGreaterThanOrEqual(6); + }); +}); diff --git a/src/services/graph/CentralityScorer.ts b/src/services/graph/CentralityScorer.ts new file mode 100644 index 0000000..edd371a --- /dev/null +++ b/src/services/graph/CentralityScorer.ts @@ -0,0 +1,43 @@ +const MIN_SIZE = 1; +const MAX_SIZE = 10; + +/** Used when every note has the same degree. There's nothing to compare, so all nodes get the same mid-range size. */ +const FLAT_DEGREE_SIZE = 5; + +export class CentralityScorer { + /** + * Maps each note's degree (connection count) to a 1-10 size scale, so + * the most connected notes render biggest. See scale() for why this + * isn't plain min-max. + */ + public score(degreeMap: Map): Map { + if (degreeMap.size === 0) { + return new Map(); + } + + let min = Infinity; + let max = -Infinity; + for (const degree of degreeMap.values()) { + if (degree < min) min = degree; + if (degree > max) max = degree; + } + const spread = max - min; + + const sizes = new Map(); + for (const [noteId, degree] of degreeMap) { + sizes.set(noteId, spread === 0 ? FLAT_DEGREE_SIZE : this.scale(degree, min, spread)); + } + return sizes; + } + + /** + * Most notes only have a few connections, and a couple of hubs have way + * more. Plain min-max scaling would squeeze almost everything down near + * MIN_SIZE. Using log compression instead spreads the low-degree notes + * out across the scale instead of flattening them. + */ + private scale(degree: number, min: number, spread: number): number { + const normalized = Math.log1p(degree - min) / Math.log1p(spread); + return Math.round(MIN_SIZE + normalized * (MAX_SIZE - MIN_SIZE)); + } +} diff --git a/src/services/graph/GraphBuilder.test.ts b/src/services/graph/GraphBuilder.test.ts index 06af347..92d8c5a 100644 --- a/src/services/graph/GraphBuilder.test.ts +++ b/src/services/graph/GraphBuilder.test.ts @@ -1,13 +1,19 @@ import { GraphBuilder } from './GraphBuilder'; import { EdgeFactory } from '../similarity/EdgeFactory'; import { SimilarityEngine } from '../similarity/SimilarityEngine'; +import { LouvainDetector } from './LouvainDetector'; +import { CentralityScorer } from './CentralityScorer'; import { Note } from '../../data/Types'; jest.mock('../similarity/EdgeFactory'); jest.mock('../similarity/SimilarityEngine'); +jest.mock('./LouvainDetector'); +jest.mock('./CentralityScorer'); const MockEdgeFactory = EdgeFactory as jest.MockedClass; const MockSimilarityEngine = SimilarityEngine as jest.MockedClass; +const MockLouvainDetector = LouvainDetector as jest.MockedClass; +const MockCentralityScorer = CentralityScorer as jest.MockedClass; function note(id: string, title: string, links: string[] = []): Note { return { @@ -25,11 +31,17 @@ function note(id: string, title: string, links: string[] = []): Note { describe('GraphBuilder', () => { let builder: GraphBuilder; let mockEdgeFactory: jest.Mocked; + let mockLouvainDetector: jest.Mocked; + let mockCentralityScorer: jest.Mocked; beforeEach(() => { jest.clearAllMocks(); mockEdgeFactory = new MockEdgeFactory() as jest.Mocked; - builder = new GraphBuilder(mockEdgeFactory); + mockLouvainDetector = new MockLouvainDetector() as jest.Mocked; + mockCentralityScorer = new MockCentralityScorer() as jest.Mocked; + mockLouvainDetector.detectCommunities.mockReturnValue(new Map()); + mockCentralityScorer.score.mockReturnValue(new Map()); + builder = new GraphBuilder(mockEdgeFactory, mockLouvainDetector, mockCentralityScorer); }); it('creates nodes with degree 0 when no edges', () => { @@ -75,6 +87,35 @@ describe('GraphBuilder', () => { expect(result.edges[0].data).toEqual({ source: 'a', target: 'b', type: 'link' }); }); + it('applies the detected community and centrality size to each node', () => { + mockEdgeFactory.createEdges.mockReturnValue([{ source: 'a', target: 'b', type: 'link' }]); + mockLouvainDetector.detectCommunities.mockReturnValue( + new Map([ + ['a', 2], + ['b', 2], + ]) + ); + mockCentralityScorer.score.mockReturnValue( + new Map([ + ['a', 7], + ['b', 3], + ]) + ); + + const notes = [note('a', 'A'), note('b', 'B')]; + const result = builder.build(notes); + + expect(result.nodes[0].data).toMatchObject({ id: 'a', community: 2, size: 7 }); + expect(result.nodes[1].data).toMatchObject({ id: 'b', community: 2, size: 3 }); + }); + + it('defaults community to 0 and size to 1 when a note is missing from either map', () => { + mockEdgeFactory.createEdges.mockReturnValue([]); + const notes = [note('a', 'A')]; + const result = builder.build(notes); + expect(result.nodes[0].data).toMatchObject({ community: 0, size: 1 }); + }); + describe('buildWithSimilarity', () => { it('adds semantic edges computed from embeddings alongside structural edges', async () => { mockEdgeFactory.createEdges.mockReturnValue([ diff --git a/src/services/graph/GraphBuilder.ts b/src/services/graph/GraphBuilder.ts index 2654fc6..a3a3a32 100644 --- a/src/services/graph/GraphBuilder.ts +++ b/src/services/graph/GraphBuilder.ts @@ -3,12 +3,22 @@ import { EdgeFactory } from '../similarity/EdgeFactory'; import { SimilarityEngine } from '../similarity/SimilarityEngine'; import { EmbeddedNote } from '../embeddings/Types'; import { GraphData, GraphEdge, GraphNode } from './types'; +import { LouvainDetector } from './LouvainDetector'; +import { CentralityScorer } from './CentralityScorer'; export class GraphBuilder { private readonly edgeFactory: EdgeFactory; - - public constructor(edgeFactory = new EdgeFactory()) { + private readonly louvainDetector: LouvainDetector; + private readonly centralityScorer: CentralityScorer; + + public constructor( + edgeFactory = new EdgeFactory(), + louvainDetector = new LouvainDetector(), + centralityScorer = new CentralityScorer() + ) { this.edgeFactory = edgeFactory; + this.louvainDetector = louvainDetector; + this.centralityScorer = centralityScorer; } /** @@ -43,12 +53,14 @@ export class GraphBuilder { private buildData(notes: Note[], edges: GraphEdge[]): GraphData { const degreeMap = this.computeDegreeMap(notes, edges); - const nodes = this.buildNodes(notes, degreeMap); + const communities = this.louvainDetector.detectCommunities(notes, edges); + const sizes = this.centralityScorer.score(degreeMap); + const nodes = this.buildNodes(notes, degreeMap, communities, sizes); const nodeIdSet = new Set(nodes.map((n) => n.data.id)); const visibleEdges = this.filterVisibleEdges(edges, nodeIdSet); - this.logGraphStats(nodes, visibleEdges, degreeMap); + this.logGraphStats(nodes, visibleEdges, degreeMap, communities); return { nodes, edges: visibleEdges.map((e) => ({ data: e })) }; } @@ -69,7 +81,12 @@ export class GraphBuilder { } /** Builds one node per note, truncating long titles to keep labels readable in the graph. */ - private buildNodes(notes: Note[], degreeMap: Map): Array<{ data: GraphNode }> { + private buildNodes( + notes: Note[], + degreeMap: Map, + communities: Map, + sizes: Map + ): Array<{ data: GraphNode }> { const nodes: Array<{ data: GraphNode }> = []; for (const note of notes) { const degree = degreeMap.get(note.id) ?? 0; @@ -80,6 +97,8 @@ export class GraphBuilder { label: label.length > 64 ? label.substring(0, 61) + '...' : label, noteId: note.id, degree, + community: communities.get(note.id) ?? 0, + size: sizes.get(note.id) ?? 1, }, }); } @@ -94,7 +113,8 @@ export class GraphBuilder { private logGraphStats( nodes: Array<{ data: GraphNode }>, visibleEdges: GraphEdge[], - degreeMap: Map + degreeMap: Map, + communities: Map ): void { const connectedIds = new Set(); for (const edge of visibleEdges) { @@ -103,10 +123,11 @@ export class GraphBuilder { } const isolatedCount = nodes.length - connectedIds.size; const maxDegree = Math.max(1, ...degreeMap.values()); + const communityCount = new Set(communities.values()).size; console.info( `Graph built: ${nodes.length} nodes, ${visibleEdges.length} edges ` + - `(${isolatedCount} isolated, max degree ${maxDegree})` + `(${isolatedCount} isolated, max degree ${maxDegree}, ${communityCount} communities)` ); } } diff --git a/src/services/graph/LouvainDetector.test.ts b/src/services/graph/LouvainDetector.test.ts new file mode 100644 index 0000000..5383038 --- /dev/null +++ b/src/services/graph/LouvainDetector.test.ts @@ -0,0 +1,195 @@ +import { LouvainDetector } from './LouvainDetector'; +import { Note } from '../../data/Types'; +import { GraphEdge } from './types'; + +function note(id: string, title: string, body = ''): Note { + return { + id, + parent_id: 'p1', + title, + body, + created_time: 0, + updated_time: 1, + links: [], + tags: [], + }; +} + +describe('LouvainDetector', () => { + let detector: LouvainDetector; + + beforeEach(() => { + detector = new LouvainDetector(); + }); + + describe('sparse fallback (keyword grouping)', () => { + it('groups notes sharing a dominant keyword into the same community', () => { + const notes = [ + note('a', 'Gardening tips', 'Watering the garden every gardening morning'), + note('b', 'More gardening', 'Gardening pruning gardening advice'), + note('c', 'Cooking basics', 'Cooking pasta cooking recipes'), + ]; + + const communities = detector.detectCommunities(notes, []); + + expect(communities.get('a')).toBe(communities.get('b')); + expect(communities.get('a')).not.toBe(communities.get('c')); + }); + + it('falls back to keyword grouping when there are fewer than 3 notes, even with edges', () => { + const notes = [note('a', 'Alpha document'), note('b', 'Beta document')]; + const edges: GraphEdge[] = [{ source: 'a', target: 'b', type: 'link' }]; + + const communities = detector.detectCommunities(notes, edges); + + expect(communities.size).toBe(2); + expect(communities.get('a')).not.toBe(communities.get('b')); + }); + + it('gives each note its own community when no keyword repeats across notes', () => { + const notes = [ + note('a', 'Zebra migration'), + note('b', 'Quantum entanglement'), + note('c', 'Symphony orchestra'), + ]; + + const communities = detector.detectCommunities(notes, []); + + expect(new Set(communities.values()).size).toBe(3); + }); + }); + + describe('Louvain', () => { + it('places directly connected notes in the same community and separates disconnected clusters', () => { + const notes = ['a', 'b', 'c', 'd', 'e', 'f'].map((id) => note(id, id)); + const edges: GraphEdge[] = [ + { source: 'a', target: 'b', type: 'link' }, + { source: 'b', target: 'c', type: 'link' }, + { source: 'a', target: 'c', type: 'link' }, + { source: 'd', target: 'e', type: 'link' }, + { source: 'e', target: 'f', type: 'link' }, + { source: 'd', target: 'f', type: 'link' }, + ]; + + const communities = detector.detectCommunities(notes, edges); + + expect(communities.get('a')).toBe(communities.get('b')); + expect(communities.get('a')).toBe(communities.get('c')); + expect(communities.get('d')).toBe(communities.get('e')); + expect(communities.get('d')).toBe(communities.get('f')); + expect(communities.get('a')).not.toBe(communities.get('d')); + }); + + it('numbers communities by descending size, ties broken by the lowest member id, so results are stable', () => { + const notes = ['a', 'b', 'c', 'd', 'e', 'f'].map((id) => note(id, id)); + const edges: GraphEdge[] = [ + { source: 'a', target: 'b', type: 'link' }, + { source: 'b', target: 'c', type: 'link' }, + { source: 'a', target: 'c', type: 'link' }, + { source: 'd', target: 'e', type: 'link' }, + { source: 'e', target: 'f', type: 'link' }, + { source: 'd', target: 'f', type: 'link' }, + ]; + + const communities = detector.detectCommunities(notes, edges); + + expect(communities.get('a')).toBe(0); + expect(communities.get('d')).toBe(1); + }); + + it('produces identical assignments across repeated runs on the same graph', () => { + const notes = ['a', 'b', 'c', 'd', 'e', 'f'].map((id) => note(id, id)); + const edges: GraphEdge[] = [ + { source: 'a', target: 'b', type: 'link' }, + { source: 'b', target: 'c', type: 'link' }, + { source: 'a', target: 'c', type: 'link' }, + { source: 'd', target: 'e', type: 'link' }, + { source: 'e', target: 'f', type: 'link' }, + { source: 'd', target: 'f', type: 'link' }, + ]; + + const first = detector.detectCommunities(notes, edges); + const second = detector.detectCommunities(notes, edges); + + expect(Array.from(second.entries())).toEqual(Array.from(first.entries())); + }); + + it('weighs a note more strongly toward a cluster it shares multiple edge types with', () => { + // x is tied to triangle A by two relationships (a link and a tag + // between the same pair) and to triangle B by a single link. + // Tested this against the real library. Without adding up the + // weight per edge type, x-a1 and x-b1 both stay at weight 1 and x + // ties toward B. With the weight added up, x-a1 reaches weight 2 + // and pulls x into A instead. This is a real regression test for + // that logic, not just a "doesn't throw" check. + const triangle = (prefix: string): GraphEdge[] => [ + { source: `${prefix}1`, target: `${prefix}2`, type: 'link' }, + { source: `${prefix}2`, target: `${prefix}3`, type: 'link' }, + { source: `${prefix}1`, target: `${prefix}3`, type: 'link' }, + ]; + const notes = ['x', 'a1', 'a2', 'a3', 'b1', 'b2', 'b3'].map((id) => note(id, id)); + const edges: GraphEdge[] = [ + ...triangle('a'), + ...triangle('b'), + { source: 'x', target: 'a1', type: 'link' }, + { source: 'x', target: 'a1', type: 'tag' }, + { source: 'x', target: 'b1', type: 'link' }, + ]; + + const communities = detector.detectCommunities(notes, edges); + + expect(communities.get('x')).toBe(communities.get('a1')); + expect(communities.get('x')).not.toBe(communities.get('b1')); + }); + + it('assigns every note a community, including notes with no edges of their own', () => { + const notes = ['a', 'b', 'c', 'd', 'isolated'].map((id) => note(id, id)); + const edges: GraphEdge[] = [ + { source: 'a', target: 'b', type: 'link' }, + { source: 'b', target: 'c', type: 'link' }, + { source: 'a', target: 'c', type: 'link' }, + { source: 'c', target: 'd', type: 'link' }, + ]; + + const communities = detector.detectCommunities(notes, edges); + + expect(communities.size).toBe(notes.length); + expect(communities.has('isolated')).toBe(true); + }); + + it('does not throw on edges referencing notes outside the note set', () => { + const notes = [note('a', 'A'), note('b', 'B'), note('c', 'C')]; + const edges: GraphEdge[] = [ + { source: 'a', target: 'b', type: 'link' }, + { source: 'b', target: 'c', type: 'link' }, + { source: 'a', target: 'missing', type: 'link' }, + ]; + + expect(() => detector.detectCommunities(notes, edges)).not.toThrow(); + }); + + it('falls back to keyword grouping when Louvain resolves to near-all singletons', () => { + const notes = [ + note('a', 'Linked one'), + note('b', 'Linked two'), + note('c', 'Gardening tips', 'Gardening advice gardening'), + note('d', 'More gardening', 'Gardening notes gardening tips'), + note('e1', 'Isolate one'), + note('e2', 'Isolate two'), + note('e3', 'Isolate three'), + note('e4', 'Isolate four'), + note('e5', 'Isolate five'), + note('e6', 'Isolate six'), + ]; + // Just one edge among 10 otherwise disconnected notes. Louvain + // would end up with 9 communities (one pair plus eight + // singletons), well past the degenerate threshold, even though + // edges.length is greater than 0. + const edges: GraphEdge[] = [{ source: 'a', target: 'b', type: 'link' }]; + + const communities = detector.detectCommunities(notes, edges); + + expect(communities.get('c')).toBe(communities.get('d')); + }); + }); +}); diff --git a/src/services/graph/LouvainDetector.ts b/src/services/graph/LouvainDetector.ts new file mode 100644 index 0000000..4ae0bb8 --- /dev/null +++ b/src/services/graph/LouvainDetector.ts @@ -0,0 +1,218 @@ +import Graph from 'graphology'; +import louvain from 'graphology-communities-louvain'; +import { Note } from '../../data/Types'; +import { GraphEdge } from './types'; + +/** Below this note count there isn't enough structure for Louvain to produce a meaningful result. */ +const MIN_NOTES_FOR_LOUVAIN = 3; + +/** If Louvain ends up with this many communities per note or more, it hasn't found real structure (for example one stray edge in an otherwise disconnected graph). Grouping by keyword works better than near-all-singleton clusters in that case. */ +const DEGENERATE_COMMUNITY_RATIO = 0.8; + +/** + * Seeded PRNG (mulberry32) so the same graph always produces the same + * Louvain result. The library's default `rng` is `Math.random`, which would + * otherwise reshuffle community ids, and node colors, on every rebuild of + * the same graph. + */ +const createDeterministicRng = (): (() => number) => { + let state = 0x9e3779b9; + return () => { + state = (state + 0x6d2b79f5) | 0; + let t = Math.imul(state ^ (state >>> 15), 1 | state); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +}; + +const STOPWORDS = new Set([ + 'this', + 'that', + 'these', + 'those', + 'with', + 'from', + 'have', + 'were', + 'been', + 'being', + 'about', + 'into', + 'over', + 'under', + 'again', + 'there', + 'their', + 'they', + 'them', + 'then', + 'than', + 'when', + 'what', + 'which', + 'while', + 'where', + 'your', + 'yours', + 'will', + 'would', + 'could', + 'should', + 'note', + 'notes', + 'today', + 'just', + 'also', + 'here', + 'some', + 'each', + 'more', + 'most', + 'other', + 'such', + 'only', + 'same', +]); + +/** + * Assigns each note to a community. Runs Louvain clustering on the note + * graph when it's dense enough to give a meaningful result, and falls back + * to grouping notes by their most frequent keyword otherwise. + */ +export class LouvainDetector { + /** + * When `isDegenerate` is true, the whole Louvain result gets thrown away + * for keyword grouping, even notes that were genuinely well connected. We + * could keep Louvain's real clusters and only keyword-group the + * singletons, but that adds real complexity: merging two id schemes and + * deciding how they're colored relative to each other. This case is + * already the sparse, low-signal tail end of the data, so it's kept + * all-or-nothing for now. + */ + public detectCommunities(notes: Note[], edges: GraphEdge[]): Map { + if (this.isTooSparse(notes, edges)) { + return this.groupByKeyword(notes); + } + + const raw = this.runLouvain(notes, edges); + if (this.isDegenerate(raw, notes.length)) { + return this.groupByKeyword(notes); + } + return this.renumberBySize(raw); + } + + /** Too few notes, or no connections at all, means Louvain would only produce singleton communities. */ + private isTooSparse(notes: Note[], edges: GraphEdge[]): boolean { + return notes.length < MIN_NOTES_FOR_LOUVAIN || edges.length === 0; + } + + /** Catches something a raw edge count can't: a few edges scattered across an otherwise disconnected graph, like a strict similarity threshold. Louvain resolves that to almost all singletons. */ + private isDegenerate(raw: Record, noteCount: number): boolean { + const communityCount = new Set(Object.values(raw)).size; + return communityCount >= noteCount * DEGENERATE_COMMUNITY_RATIO; + } + + /** + * Builds the graphology graph, weighting each edge by how many different + * relationships connect the same pair of notes. Two notes that are both + * linked and semantically similar are a stronger pair than two notes that + * just happen to share a tag. Louvain reads this through its 'weight' + * edge attribute by default, so adding up the weight here, instead of + * collapsing every relationship into one unweighted edge, lets strongly + * related notes end up in the same community more easily. + */ + private runLouvain(notes: Note[], edges: GraphEdge[]): Record { + const graph = new Graph({ type: 'undirected' }); + for (const note of notes) { + graph.addNode(note.id); + } + for (const edge of edges) { + if (!graph.hasNode(edge.source) || !graph.hasNode(edge.target)) { + continue; + } + if (graph.hasEdge(edge.source, edge.target)) { + graph.updateEdgeAttribute(edge.source, edge.target, 'weight', (w) => (w ?? 1) + 1); + } else { + graph.mergeEdge(edge.source, edge.target, { weight: 1 }); + } + } + + return louvain(graph, { rng: createDeterministicRng() }); + } + + /** + * Louvain's raw community ids are arbitrary. Renumbering by community + * size, largest first, and breaking ties by the lowest member note id, + * gives stable and meaningful ids. Id 0 is always the largest cluster, so + * the UI can save its most distinct colors for the communities that + * matter most. + */ + private renumberBySize(raw: Record): Map { + const membersByRawId = new Map(); + for (const [noteId, rawId] of Object.entries(raw)) { + const members = membersByRawId.get(rawId); + if (members) { + members.push(noteId); + } else { + membersByRawId.set(rawId, [noteId]); + } + } + + const groups = Array.from(membersByRawId.values()).map((members) => ({ + members, + minId: members.reduce((min, id) => (id < min ? id : min)), + })); + groups.sort((a, b) => b.members.length - a.members.length || (a.minId < b.minId ? -1 : 1)); + + const renumbered = new Map(); + groups.forEach(({ members }, newId) => { + for (const noteId of members) { + renumbered.set(noteId, newId); + } + }); + return renumbered; + } + + /** Groups notes sharing the same dominant keyword into the same community. */ + private groupByKeyword(notes: Note[]): Map { + const communityByKeyword = new Map(); + const assignments = new Map(); + + for (const note of notes) { + const keyword = this.extractKeyword(note); + let community = communityByKeyword.get(keyword); + if (community === undefined) { + community = communityByKeyword.size; + communityByKeyword.set(keyword, community); + } + assignments.set(note.id, community); + } + + return assignments; + } + + /** Picks the note's own most frequent significant word, weighting the title over the body. Falls back to a note-unique key when nothing qualifies, so unrelated notes never collide. */ + private extractKeyword(note: Note): string { + const counts = new Map(); + for (const word of this.tokenize(`${note.title} ${note.title} ${note.body ?? ''}`)) { + if (STOPWORDS.has(word) || word.length <= 3) continue; + counts.set(word, (counts.get(word) ?? 0) + 1); + } + + let bestWord: string | null = null; + let bestCount = 0; + for (const [word, count] of counts) { + if (count > bestCount) { + bestWord = word; + bestCount = count; + } + } + + return bestWord ?? `note:${note.id}`; + } + + /** Only matches Latin-script words. Notes in other scripts always miss and fall through to `extractKeyword`'s per-note key. This is a known limit of this fallback-of-a-fallback path. */ + private tokenize(text: string): string[] { + return text.toLowerCase().match(/[a-z]{2,}/g) ?? []; + } +} diff --git a/src/services/graph/types.ts b/src/services/graph/types.ts index 8ab0362..54e9fec 100644 --- a/src/services/graph/types.ts +++ b/src/services/graph/types.ts @@ -6,6 +6,8 @@ export interface GraphNode { label: string; noteId: string; degree: number; + community: number; + size: number; } export interface GraphEdge { diff --git a/src/ui/graph-view.js b/src/ui/graph-view.js index de63e51..8cffd9d 100644 --- a/src/ui/graph-view.js +++ b/src/ui/graph-view.js @@ -17,7 +17,9 @@ var FCOSE_OPTIONS = { uniformNodeDimensions: true, packComponents: true, nodeSeparation: 140, - nodeRepulsion: function () { return 8000; }, + nodeRepulsion: function () { + return 8000; + }, gravity: 0.12, gravityRange: 5.0, idealEdgeLength: 180, @@ -66,6 +68,41 @@ function hideProgress() { } } +/** + * Colors for community groups. Community ids are ordered largest first, so + * index 0 is always the biggest cluster. The first 7 colors are the + * Okabe-Ito colorblind-safe palette. 3 more were added and kept as distinct + * as possible, since Okabe-Ito only covers 7 usable colors and we need 10. + */ +var COMMUNITY_COLORS = [ + '#e69f00', // orange + '#56b4e9', // sky blue + '#009e73', // bluish green + '#f0e442', // yellow + '#0072b2', // blue + '#d55e00', // vermillion + '#cc79a7', // reddish purple + '#332288', // indigo + '#44aa99', // teal + '#aa4499', // purple +]; + +/** Neutral color for communities past the palette. A long tail of small groups isn't worth giving each one its own color. */ +var COMMUNITY_OVERFLOW_COLOR = '#9aa0a6'; + +/** Maps a node's community id onto the categorical palette. */ +function communityColor(ele) { + var community = ele.data('community') || 0; + if (community >= COMMUNITY_COLORS.length) return COMMUNITY_OVERFLOW_COLOR; + return COMMUNITY_COLORS[community]; +} + +/** Turns a node's 1-10 centrality score into a pixel size, so small notes stay readable and hubs stand out. */ +function nodeDiameter(ele) { + var size = ele.data('size') || 1; + return 18 + (size - 1) * 3; +} + /** Detect whether the current Joplin theme is dark by computing luminance of --joplin-background-color. */ function isDarkTheme() { var bg = getComputedStyle(document.body).getPropertyValue('--joplin-background-color').trim(); @@ -86,7 +123,7 @@ function buildStylesheet() { { selector: 'node', style: { - 'background-color': '#5b9bd5', + 'background-color': communityColor, label: 'data(label)', color: dark ? '#ddd' : '#222', 'font-size': '9px', @@ -95,10 +132,10 @@ function buildStylesheet() { 'text-margin-y': -4, 'text-wrap': 'ellipsis', 'text-max-width': '100px', - width: 28, - height: 28, + width: nodeDiameter, + height: nodeDiameter, 'border-width': 1.5, - 'border-color': '#4a8cc4', + 'border-color': dark ? '#1e1e1e' : '#ffffff', }, }, { @@ -245,7 +282,8 @@ function updateStats(notes, explicit, semantic, tags) { function createExportMenu(btn) { var menu = document.createElement('div'); menu.className = 'export-menu'; - menu.innerHTML = ''; + menu.innerHTML = + ''; document.body.appendChild(menu); btn.addEventListener('click', function (e) { @@ -255,7 +293,7 @@ function createExportMenu(btn) { if (!open) { var rect = btn.getBoundingClientRect(); menu.style.left = rect.left + 'px'; - menu.style.top = (rect.bottom + 4) + 'px'; + menu.style.top = rect.bottom + 4 + 'px'; } }); @@ -265,7 +303,9 @@ function createExportMenu(btn) { if (!item) return; var format = item.getAttribute('data-format'); menu.style.display = 'none'; - var bg = getComputedStyle(document.body).getPropertyValue('--joplin-background-color').trim() || '#1e1e1e'; + var bg = + getComputedStyle(document.body).getPropertyValue('--joplin-background-color').trim() || + '#1e1e1e'; if (format === 'png') { downloadFile(cy.png({ full: true, bg: bg }), 'note-graph.png'); } else if (format === 'svg') { @@ -273,7 +313,9 @@ function createExportMenu(btn) { var svgBlob = new Blob([svgString], { type: 'image/svg+xml;charset=utf-8' }); downloadFile(URL.createObjectURL(svgBlob), 'note-graph.svg'); } else if (format === 'json') { - var blob = new Blob([JSON.stringify(cy.json().elements, null, 2)], { type: 'application/json' }); + var blob = new Blob([JSON.stringify(cy.json().elements, null, 2)], { + type: 'application/json', + }); downloadFile(URL.createObjectURL(blob), 'note-graph.json'); } }); @@ -327,7 +369,7 @@ function init() { var headerH = header ? header.offsetHeight : 0; var legendH = legend ? legend.offsetHeight : 0; var statsH = statsBar ? statsBar.offsetHeight : 0; - container.style.height = (window.innerHeight - headerH - legendH - statsH) + 'px'; + container.style.height = window.innerHeight - headerH - legendH - statsH + 'px'; container.style.minHeight = '350px'; container.style.width = '100%'; @@ -365,7 +407,10 @@ function init() { zoomInBtn.addEventListener('click', function () { cy.zoom({ level: cy.zoom() * 1.3, - renderedPosition: { x: container.clientWidth / 2, y: container.clientHeight / 2 }, + renderedPosition: { + x: container.clientWidth / 2, + y: container.clientHeight / 2, + }, }); }); } @@ -373,7 +418,10 @@ function init() { zoomOutBtn.addEventListener('click', function () { cy.zoom({ level: cy.zoom() * 0.7, - renderedPosition: { x: container.clientWidth / 2, y: container.clientHeight / 2 }, + renderedPosition: { + x: container.clientWidth / 2, + y: container.clientHeight / 2, + }, }); }); } @@ -389,8 +437,8 @@ function init() { cy.on('mousemove', 'edge[type="tag"]', function (evt) { if (!tooltipEl) return; - tooltipEl.style.left = (evt.originalEvent.clientX + 12) + 'px'; - tooltipEl.style.top = (evt.originalEvent.clientY + 12) + 'px'; + tooltipEl.style.left = evt.originalEvent.clientX + 12 + 'px'; + tooltipEl.style.top = evt.originalEvent.clientY + 12 + 'px'; }); cy.on('mouseout', 'edge[type="tag"]', function () { @@ -404,19 +452,35 @@ function init() { var label = node.data('label') || '(untitled)'; var id = node.id(); var degree = node.data('degree') || 0; + var community = node.data('community') || 0; var stats = nodeStats && nodeStats[id] ? nodeStats[id] : { linkCount: 0, tagCount: 0 }; - var safeLabel = label.replace(/&/g,'&').replace(//g,'>'); - tooltipEl.innerHTML = '
' + safeLabel + '
' - + '
Degree' + degree + '
' - + '
Links' + stats.linkCount + '
' - + '
Tags' + stats.tagCount + '
'; + var safeLabel = label + .replace(/&/g, '&') + .replace(//g, '>'); + tooltipEl.innerHTML = + '
' + + safeLabel + + '
' + + '
Degree' + + degree + + '
' + + '
Links' + + stats.linkCount + + '
' + + '
Tags' + + stats.tagCount + + '
' + + '
Community' + + community + + '
'; tooltipEl.style.display = 'block'; }); cy.on('mousemove', 'node', function (evt) { if (!tooltipEl) return; - tooltipEl.style.left = (evt.originalEvent.clientX + 14) + 'px'; - tooltipEl.style.top = (evt.originalEvent.clientY + 14) + 'px'; + tooltipEl.style.left = evt.originalEvent.clientX + 14 + 'px'; + tooltipEl.style.top = evt.originalEvent.clientY + 14 + 'px'; }); cy.on('mouseout', 'node', function () { @@ -434,23 +498,33 @@ function init() { var h = header ? header.offsetHeight : 0; var lh = legend ? legend.offsetHeight : 0; var sh = statsBar ? statsBar.offsetHeight : 0; - container.style.height = (window.innerHeight - h - lh - sh) + 'px'; + container.style.height = window.innerHeight - h - lh - sh + 'px'; cy.resize(); cy.fit(undefined, 30); }); observer.observe(container); observer.observe(document.body); - var lastBg = getComputedStyle(document.body).getPropertyValue('--joplin-background-color').trim(); + var lastBg = getComputedStyle(document.body) + .getPropertyValue('--joplin-background-color') + .trim(); var themeObserver = new MutationObserver(function () { - var currentBg = getComputedStyle(document.body).getPropertyValue('--joplin-background-color').trim(); + var currentBg = getComputedStyle(document.body) + .getPropertyValue('--joplin-background-color') + .trim(); if (currentBg !== lastBg) { lastBg = currentBg; cy.style().fromJson(buildStylesheet()).update(); } }); - themeObserver.observe(document.documentElement, { attributes: true, attributeFilter: ['style', 'class'] }); - themeObserver.observe(document.body, { attributes: true, attributeFilter: ['style', 'class'] }); + themeObserver.observe(document.documentElement, { + attributes: true, + attributeFilter: ['style', 'class'], + }); + themeObserver.observe(document.body, { + attributes: true, + attributeFilter: ['style', 'class'], + }); var fitBtn = document.getElementById('graph-fit'); if (fitBtn) { @@ -484,8 +558,7 @@ function init() { var q = this.value.trim().toLowerCase(); if (searchTimer) clearTimeout(searchTimer); cy.nodes().style('opacity', 1); - cy.nodes().style('border-width', 1.5); - cy.nodes().style('border-color', '#4a8cc4'); + cy.nodes().removeStyle('border-width border-color'); cy.nodes().stop(true, false); if (!q) return; cy.nodes().style('opacity', 0.15); @@ -497,8 +570,7 @@ function init() { matches.style('border-width', 3); matches.style('border-color', '#ffa500'); searchTimer = setTimeout(function () { - matches.style('border-width', 1.5); - matches.style('border-color', '#4a8cc4'); + matches.removeStyle('border-width border-color'); }, 800); cy.animate({ fit: { eles: matches, padding: 50 }, duration: 400 }); } diff --git a/tsconfig.json b/tsconfig.json index 2120989..70f2414 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -5,6 +5,7 @@ "target": "es2015", "jsx": "react", "allowJs": true, + "esModuleInterop": true, "baseUrl": ".", "ignoreDeprecations": "6.0", "types": ["jest", "node"] From 8a465b54db7430a8c741aad5b4e143ac34084dff Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Mon, 27 Jul 2026 23:18:34 +0530 Subject: [PATCH 25/28] ANG-010: Cleaned code --- src/services/graph/CentralityScorer.test.ts | 4 +- src/services/graph/CentralityScorer.ts | 13 +------ src/services/graph/LouvainDetector.test.ts | 15 ++----- src/services/graph/LouvainDetector.ts | 43 ++++----------------- src/ui/graph-view.js | 10 +---- 5 files changed, 16 insertions(+), 69 deletions(-) diff --git a/src/services/graph/CentralityScorer.test.ts b/src/services/graph/CentralityScorer.test.ts index bf65013..ccb5522 100644 --- a/src/services/graph/CentralityScorer.test.ts +++ b/src/services/graph/CentralityScorer.test.ts @@ -48,7 +48,6 @@ describe('CentralityScorer', () => { }); it('spreads a right-skewed degree distribution instead of pinning most notes near the minimum', () => { - // A realistic shape: most notes are low (3-8), one hub is far above them. const result = scorer.score( new Map([ ['a', 0], @@ -61,8 +60,7 @@ describe('CentralityScorer', () => { ); expect(result.get('a')).toBe(1); expect(result.get('hub')).toBe(10); - // Plain min-max would put all of these at size 1-2. The log curve - // should spread them further up the scale. + // Plain min-max would leave these near size 1-2. expect(result.get('b')).toBeGreaterThanOrEqual(4); expect(result.get('e')).toBeGreaterThanOrEqual(6); }); diff --git a/src/services/graph/CentralityScorer.ts b/src/services/graph/CentralityScorer.ts index edd371a..397b087 100644 --- a/src/services/graph/CentralityScorer.ts +++ b/src/services/graph/CentralityScorer.ts @@ -5,11 +5,7 @@ const MAX_SIZE = 10; const FLAT_DEGREE_SIZE = 5; export class CentralityScorer { - /** - * Maps each note's degree (connection count) to a 1-10 size scale, so - * the most connected notes render biggest. See scale() for why this - * isn't plain min-max. - */ + /** Maps each note's degree to a 1-10 size scale. See `scale()` for why this isn't plain min-max. */ public score(degreeMap: Map): Map { if (degreeMap.size === 0) { return new Map(); @@ -30,12 +26,7 @@ export class CentralityScorer { return sizes; } - /** - * Most notes only have a few connections, and a couple of hubs have way - * more. Plain min-max scaling would squeeze almost everything down near - * MIN_SIZE. Using log compression instead spreads the low-degree notes - * out across the scale instead of flattening them. - */ + /** Log compression instead of plain min-max, since most notes have few connections and a couple of hubs have way more; linear scaling would squeeze everyone but the hubs down near MIN_SIZE. */ private scale(degree: number, min: number, spread: number): number { const normalized = Math.log1p(degree - min) / Math.log1p(spread); return Math.round(MIN_SIZE + normalized * (MAX_SIZE - MIN_SIZE)); diff --git a/src/services/graph/LouvainDetector.test.ts b/src/services/graph/LouvainDetector.test.ts index 5383038..b072014 100644 --- a/src/services/graph/LouvainDetector.test.ts +++ b/src/services/graph/LouvainDetector.test.ts @@ -115,13 +115,9 @@ describe('LouvainDetector', () => { }); it('weighs a note more strongly toward a cluster it shares multiple edge types with', () => { - // x is tied to triangle A by two relationships (a link and a tag - // between the same pair) and to triangle B by a single link. - // Tested this against the real library. Without adding up the - // weight per edge type, x-a1 and x-b1 both stay at weight 1 and x - // ties toward B. With the weight added up, x-a1 reaches weight 2 - // and pulls x into A instead. This is a real regression test for - // that logic, not just a "doesn't throw" check. + // x has two relationships with a1 (link + tag) but only one with b1. + // Verified against the real library that this specific setup is what + // flips x from tying toward b1 to grouping with a1. const triangle = (prefix: string): GraphEdge[] => [ { source: `${prefix}1`, target: `${prefix}2`, type: 'link' }, { source: `${prefix}2`, target: `${prefix}3`, type: 'link' }, @@ -181,10 +177,7 @@ describe('LouvainDetector', () => { note('e5', 'Isolate five'), note('e6', 'Isolate six'), ]; - // Just one edge among 10 otherwise disconnected notes. Louvain - // would end up with 9 communities (one pair plus eight - // singletons), well past the degenerate threshold, even though - // edges.length is greater than 0. + // One edge among 10 otherwise disconnected notes: 9 communities, past the degenerate threshold. const edges: GraphEdge[] = [{ source: 'a', target: 'b', type: 'link' }]; const communities = detector.detectCommunities(notes, edges); diff --git a/src/services/graph/LouvainDetector.ts b/src/services/graph/LouvainDetector.ts index 4ae0bb8..e8c4899 100644 --- a/src/services/graph/LouvainDetector.ts +++ b/src/services/graph/LouvainDetector.ts @@ -6,15 +6,10 @@ import { GraphEdge } from './types'; /** Below this note count there isn't enough structure for Louvain to produce a meaningful result. */ const MIN_NOTES_FOR_LOUVAIN = 3; -/** If Louvain ends up with this many communities per note or more, it hasn't found real structure (for example one stray edge in an otherwise disconnected graph). Grouping by keyword works better than near-all-singleton clusters in that case. */ +/** At or above this ratio of communities to notes, Louvain has basically found nothing (near-all singletons). */ const DEGENERATE_COMMUNITY_RATIO = 0.8; -/** - * Seeded PRNG (mulberry32) so the same graph always produces the same - * Louvain result. The library's default `rng` is `Math.random`, which would - * otherwise reshuffle community ids, and node colors, on every rebuild of - * the same graph. - */ +/** Seeded PRNG so the same graph always produces the same Louvain result, instead of the library's default `Math.random` reshuffling colors on every rebuild. */ const createDeterministicRng = (): (() => number) => { let state = 0x9e3779b9; return () => { @@ -80,15 +75,7 @@ const STOPWORDS = new Set([ * to grouping notes by their most frequent keyword otherwise. */ export class LouvainDetector { - /** - * When `isDegenerate` is true, the whole Louvain result gets thrown away - * for keyword grouping, even notes that were genuinely well connected. We - * could keep Louvain's real clusters and only keyword-group the - * singletons, but that adds real complexity: merging two id schemes and - * deciding how they're colored relative to each other. This case is - * already the sparse, low-signal tail end of the data, so it's kept - * all-or-nothing for now. - */ + /** Degenerate results are discarded entirely rather than partially kept, on purpose, to avoid mixing two different id schemes. */ public detectCommunities(notes: Note[], edges: GraphEdge[]): Map { if (this.isTooSparse(notes, edges)) { return this.groupByKeyword(notes); @@ -106,21 +93,12 @@ export class LouvainDetector { return notes.length < MIN_NOTES_FOR_LOUVAIN || edges.length === 0; } - /** Catches something a raw edge count can't: a few edges scattered across an otherwise disconnected graph, like a strict similarity threshold. Louvain resolves that to almost all singletons. */ private isDegenerate(raw: Record, noteCount: number): boolean { const communityCount = new Set(Object.values(raw)).size; return communityCount >= noteCount * DEGENERATE_COMMUNITY_RATIO; } - /** - * Builds the graphology graph, weighting each edge by how many different - * relationships connect the same pair of notes. Two notes that are both - * linked and semantically similar are a stronger pair than two notes that - * just happen to share a tag. Louvain reads this through its 'weight' - * edge attribute by default, so adding up the weight here, instead of - * collapsing every relationship into one unweighted edge, lets strongly - * related notes end up in the same community more easily. - */ + /** Weights each edge by how many relationships connect the same pair of notes, so a note linked and tagged and semantically similar to another counts for more than a single coincidental edge. */ private runLouvain(notes: Note[], edges: GraphEdge[]): Record { const graph = new Graph({ type: 'undirected' }); for (const note of notes) { @@ -140,13 +118,7 @@ export class LouvainDetector { return louvain(graph, { rng: createDeterministicRng() }); } - /** - * Louvain's raw community ids are arbitrary. Renumbering by community - * size, largest first, and breaking ties by the lowest member note id, - * gives stable and meaningful ids. Id 0 is always the largest cluster, so - * the UI can save its most distinct colors for the communities that - * matter most. - */ + /** Louvain's raw ids are arbitrary. Renumbering by size (largest first, ties broken by lowest member id) makes id 0 always the biggest cluster. */ private renumberBySize(raw: Record): Map { const membersByRawId = new Map(); for (const [noteId, rawId] of Object.entries(raw)) { @@ -173,7 +145,6 @@ export class LouvainDetector { return renumbered; } - /** Groups notes sharing the same dominant keyword into the same community. */ private groupByKeyword(notes: Note[]): Map { const communityByKeyword = new Map(); const assignments = new Map(); @@ -191,7 +162,7 @@ export class LouvainDetector { return assignments; } - /** Picks the note's own most frequent significant word, weighting the title over the body. Falls back to a note-unique key when nothing qualifies, so unrelated notes never collide. */ + /** Falls back to a note-unique key when nothing qualifies, so unrelated notes never collide. */ private extractKeyword(note: Note): string { const counts = new Map(); for (const word of this.tokenize(`${note.title} ${note.title} ${note.body ?? ''}`)) { @@ -211,7 +182,7 @@ export class LouvainDetector { return bestWord ?? `note:${note.id}`; } - /** Only matches Latin-script words. Notes in other scripts always miss and fall through to `extractKeyword`'s per-note key. This is a known limit of this fallback-of-a-fallback path. */ + /** Latin-script words only; other scripts fall through to the per-note key above. */ private tokenize(text: string): string[] { return text.toLowerCase().match(/[a-z]{2,}/g) ?? []; } diff --git a/src/ui/graph-view.js b/src/ui/graph-view.js index 8cffd9d..a0a6187 100644 --- a/src/ui/graph-view.js +++ b/src/ui/graph-view.js @@ -68,12 +68,7 @@ function hideProgress() { } } -/** - * Colors for community groups. Community ids are ordered largest first, so - * index 0 is always the biggest cluster. The first 7 colors are the - * Okabe-Ito colorblind-safe palette. 3 more were added and kept as distinct - * as possible, since Okabe-Ito only covers 7 usable colors and we need 10. - */ +/** Colors for community groups, ordered largest cluster first. First 7 are the Okabe-Ito colorblind-safe palette, 3 more added to reach 10. */ var COMMUNITY_COLORS = [ '#e69f00', // orange '#56b4e9', // sky blue @@ -90,14 +85,13 @@ var COMMUNITY_COLORS = [ /** Neutral color for communities past the palette. A long tail of small groups isn't worth giving each one its own color. */ var COMMUNITY_OVERFLOW_COLOR = '#9aa0a6'; -/** Maps a node's community id onto the categorical palette. */ function communityColor(ele) { var community = ele.data('community') || 0; if (community >= COMMUNITY_COLORS.length) return COMMUNITY_OVERFLOW_COLOR; return COMMUNITY_COLORS[community]; } -/** Turns a node's 1-10 centrality score into a pixel size, so small notes stay readable and hubs stand out. */ +/** Maps the 1-10 centrality score to a pixel diameter. */ function nodeDiameter(ele) { var size = ele.data('size') || 1; return 18 + (size - 1) * 3; From d726a726aa9ed31bf4fb5c8d51f0c0a0f5e5df68 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Tue, 28 Jul 2026 13:14:10 +0530 Subject: [PATCH 26/28] ANG-010:Fix community color ordering for keyword fallback --- src/services/graph/LouvainDetector.test.ts | 20 ++++++ src/services/graph/LouvainDetector.ts | 18 +++-- src/ui/graph-view.js | 83 ++++++---------------- 3 files changed, 54 insertions(+), 67 deletions(-) diff --git a/src/services/graph/LouvainDetector.test.ts b/src/services/graph/LouvainDetector.test.ts index b072014..bb69c44 100644 --- a/src/services/graph/LouvainDetector.test.ts +++ b/src/services/graph/LouvainDetector.test.ts @@ -57,6 +57,26 @@ describe('LouvainDetector', () => { expect(new Set(communities.values()).size).toBe(3); }); + + it('numbers keyword-fallback communities by descending size, not by first-appearance order', () => { + const notes = [ + note('solo1', 'Astronomy basics'), + note('solo2', 'Philosophy overview'), + note('a', 'Gardening tips', 'Watering the garden every gardening morning'), + note('b', 'More gardening', 'Gardening pruning gardening advice'), + note('c', 'Gardening again', 'Gardening season gardening harvest'), + ]; + + const communities = detector.detectCommunities(notes, []); + + // The 3-note gardening group is the largest, so it must get id 0 even though + // it appears after the two singleton notes in the input. + expect(communities.get('a')).toBe(0); + expect(communities.get('b')).toBe(0); + expect(communities.get('c')).toBe(0); + expect(communities.get('solo1')).not.toBe(0); + expect(communities.get('solo2')).not.toBe(0); + }); }); describe('Louvain', () => { diff --git a/src/services/graph/LouvainDetector.ts b/src/services/graph/LouvainDetector.ts index e8c4899..566d9a4 100644 --- a/src/services/graph/LouvainDetector.ts +++ b/src/services/graph/LouvainDetector.ts @@ -75,17 +75,21 @@ const STOPWORDS = new Set([ * to grouping notes by their most frequent keyword otherwise. */ export class LouvainDetector { - /** Degenerate results are discarded entirely rather than partially kept, on purpose, to avoid mixing two different id schemes. */ + /** + * Both the Louvain and keyword-fallback paths are renumbered by size before returning, so + * callers (e.g. the community color palette) can always rely on id 0 being the largest + * community regardless of which path produced the result. + */ public detectCommunities(notes: Note[], edges: GraphEdge[]): Map { if (this.isTooSparse(notes, edges)) { - return this.groupByKeyword(notes); + return this.renumberBySize(this.groupByKeyword(notes)); } const raw = this.runLouvain(notes, edges); if (this.isDegenerate(raw, notes.length)) { - return this.groupByKeyword(notes); + return this.renumberBySize(this.groupByKeyword(notes)); } - return this.renumberBySize(raw); + return this.renumberBySize(new Map(Object.entries(raw))); } /** Too few notes, or no connections at all, means Louvain would only produce singleton communities. */ @@ -118,10 +122,10 @@ export class LouvainDetector { return louvain(graph, { rng: createDeterministicRng() }); } - /** Louvain's raw ids are arbitrary. Renumbering by size (largest first, ties broken by lowest member id) makes id 0 always the biggest cluster. */ - private renumberBySize(raw: Record): Map { + /** Raw ids (from either Louvain or the keyword fallback) are arbitrary. Renumbering by size (largest first, ties broken by lowest member id) makes id 0 always the biggest cluster. */ + private renumberBySize(raw: Map): Map { const membersByRawId = new Map(); - for (const [noteId, rawId] of Object.entries(raw)) { + for (const [noteId, rawId] of raw) { const members = membersByRawId.get(rawId); if (members) { members.push(noteId); diff --git a/src/ui/graph-view.js b/src/ui/graph-view.js index a0a6187..8683aa1 100644 --- a/src/ui/graph-view.js +++ b/src/ui/graph-view.js @@ -17,9 +17,7 @@ var FCOSE_OPTIONS = { uniformNodeDimensions: true, packComponents: true, nodeSeparation: 140, - nodeRepulsion: function () { - return 8000; - }, + nodeRepulsion: function () { return 8000; }, gravity: 0.12, gravityRange: 5.0, idealEdgeLength: 180, @@ -276,8 +274,7 @@ function updateStats(notes, explicit, semantic, tags) { function createExportMenu(btn) { var menu = document.createElement('div'); menu.className = 'export-menu'; - menu.innerHTML = - ''; + menu.innerHTML = ''; document.body.appendChild(menu); btn.addEventListener('click', function (e) { @@ -287,7 +284,7 @@ function createExportMenu(btn) { if (!open) { var rect = btn.getBoundingClientRect(); menu.style.left = rect.left + 'px'; - menu.style.top = rect.bottom + 4 + 'px'; + menu.style.top = (rect.bottom + 4) + 'px'; } }); @@ -297,9 +294,7 @@ function createExportMenu(btn) { if (!item) return; var format = item.getAttribute('data-format'); menu.style.display = 'none'; - var bg = - getComputedStyle(document.body).getPropertyValue('--joplin-background-color').trim() || - '#1e1e1e'; + var bg = getComputedStyle(document.body).getPropertyValue('--joplin-background-color').trim() || '#1e1e1e'; if (format === 'png') { downloadFile(cy.png({ full: true, bg: bg }), 'note-graph.png'); } else if (format === 'svg') { @@ -307,9 +302,7 @@ function createExportMenu(btn) { var svgBlob = new Blob([svgString], { type: 'image/svg+xml;charset=utf-8' }); downloadFile(URL.createObjectURL(svgBlob), 'note-graph.svg'); } else if (format === 'json') { - var blob = new Blob([JSON.stringify(cy.json().elements, null, 2)], { - type: 'application/json', - }); + var blob = new Blob([JSON.stringify(cy.json().elements, null, 2)], { type: 'application/json' }); downloadFile(URL.createObjectURL(blob), 'note-graph.json'); } }); @@ -363,7 +356,7 @@ function init() { var headerH = header ? header.offsetHeight : 0; var legendH = legend ? legend.offsetHeight : 0; var statsH = statsBar ? statsBar.offsetHeight : 0; - container.style.height = window.innerHeight - headerH - legendH - statsH + 'px'; + container.style.height = (window.innerHeight - headerH - legendH - statsH) + 'px'; container.style.minHeight = '350px'; container.style.width = '100%'; @@ -401,10 +394,7 @@ function init() { zoomInBtn.addEventListener('click', function () { cy.zoom({ level: cy.zoom() * 1.3, - renderedPosition: { - x: container.clientWidth / 2, - y: container.clientHeight / 2, - }, + renderedPosition: { x: container.clientWidth / 2, y: container.clientHeight / 2 }, }); }); } @@ -412,10 +402,7 @@ function init() { zoomOutBtn.addEventListener('click', function () { cy.zoom({ level: cy.zoom() * 0.7, - renderedPosition: { - x: container.clientWidth / 2, - y: container.clientHeight / 2, - }, + renderedPosition: { x: container.clientWidth / 2, y: container.clientHeight / 2 }, }); }); } @@ -431,8 +418,8 @@ function init() { cy.on('mousemove', 'edge[type="tag"]', function (evt) { if (!tooltipEl) return; - tooltipEl.style.left = evt.originalEvent.clientX + 12 + 'px'; - tooltipEl.style.top = evt.originalEvent.clientY + 12 + 'px'; + tooltipEl.style.left = (evt.originalEvent.clientX + 12) + 'px'; + tooltipEl.style.top = (evt.originalEvent.clientY + 12) + 'px'; }); cy.on('mouseout', 'edge[type="tag"]', function () { @@ -448,33 +435,19 @@ function init() { var degree = node.data('degree') || 0; var community = node.data('community') || 0; var stats = nodeStats && nodeStats[id] ? nodeStats[id] : { linkCount: 0, tagCount: 0 }; - var safeLabel = label - .replace(/&/g, '&') - .replace(//g, '>'); - tooltipEl.innerHTML = - '
' + - safeLabel + - '
' + - '
Degree' + - degree + - '
' + - '
Links' + - stats.linkCount + - '
' + - '
Tags' + - stats.tagCount + - '
' + - '
Community' + - community + - '
'; + var safeLabel = label.replace(/&/g,'&').replace(//g,'>'); + tooltipEl.innerHTML = '
' + safeLabel + '
' + + '
Degree' + degree + '
' + + '
Links' + stats.linkCount + '
' + + '
Tags' + stats.tagCount + '
' + + '
Community' + community + '
'; tooltipEl.style.display = 'block'; }); cy.on('mousemove', 'node', function (evt) { if (!tooltipEl) return; - tooltipEl.style.left = evt.originalEvent.clientX + 14 + 'px'; - tooltipEl.style.top = evt.originalEvent.clientY + 14 + 'px'; + tooltipEl.style.left = (evt.originalEvent.clientX + 14) + 'px'; + tooltipEl.style.top = (evt.originalEvent.clientY + 14) + 'px'; }); cy.on('mouseout', 'node', function () { @@ -492,33 +465,23 @@ function init() { var h = header ? header.offsetHeight : 0; var lh = legend ? legend.offsetHeight : 0; var sh = statsBar ? statsBar.offsetHeight : 0; - container.style.height = window.innerHeight - h - lh - sh + 'px'; + container.style.height = (window.innerHeight - h - lh - sh) + 'px'; cy.resize(); cy.fit(undefined, 30); }); observer.observe(container); observer.observe(document.body); - var lastBg = getComputedStyle(document.body) - .getPropertyValue('--joplin-background-color') - .trim(); + var lastBg = getComputedStyle(document.body).getPropertyValue('--joplin-background-color').trim(); var themeObserver = new MutationObserver(function () { - var currentBg = getComputedStyle(document.body) - .getPropertyValue('--joplin-background-color') - .trim(); + var currentBg = getComputedStyle(document.body).getPropertyValue('--joplin-background-color').trim(); if (currentBg !== lastBg) { lastBg = currentBg; cy.style().fromJson(buildStylesheet()).update(); } }); - themeObserver.observe(document.documentElement, { - attributes: true, - attributeFilter: ['style', 'class'], - }); - themeObserver.observe(document.body, { - attributes: true, - attributeFilter: ['style', 'class'], - }); + themeObserver.observe(document.documentElement, { attributes: true, attributeFilter: ['style', 'class'] }); + themeObserver.observe(document.body, { attributes: true, attributeFilter: ['style', 'class'] }); var fitBtn = document.getElementById('graph-fit'); if (fitBtn) { From eca9d5faebc66ffaad960baba2abf1cb223f6705 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Tue, 28 Jul 2026 23:33:48 +0530 Subject: [PATCH 27/28] ANG-010:added try-catch --- src/services/graph/GraphBuilder.test.ts | 5 +- src/services/graph/GraphBuilder.ts | 12 +- src/services/graph/LouvainDetector.test.ts | 110 +++++++++++++- src/services/graph/LouvainDetector.ts | 160 ++++++++++++++++++--- 4 files changed, 257 insertions(+), 30 deletions(-) diff --git a/src/services/graph/GraphBuilder.test.ts b/src/services/graph/GraphBuilder.test.ts index 92d8c5a..d6fcc4e 100644 --- a/src/services/graph/GraphBuilder.test.ts +++ b/src/services/graph/GraphBuilder.test.ts @@ -109,11 +109,14 @@ describe('GraphBuilder', () => { expect(result.nodes[1].data).toMatchObject({ id: 'b', community: 2, size: 3 }); }); - it('defaults community to 0 and size to 1 when a note is missing from either map', () => { + it('defaults community to 0 and size to 1 when a note is missing from either map, and logs it', () => { + const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => undefined); mockEdgeFactory.createEdges.mockReturnValue([]); const notes = [note('a', 'A')]; const result = builder.build(notes); expect(result.nodes[0].data).toMatchObject({ community: 0, size: 1 }); + expect(consoleErrorSpy).toHaveBeenCalledWith(expect.stringContaining('a')); + consoleErrorSpy.mockRestore(); }); describe('buildWithSimilarity', () => { diff --git a/src/services/graph/GraphBuilder.ts b/src/services/graph/GraphBuilder.ts index a3a3a32..16ee717 100644 --- a/src/services/graph/GraphBuilder.ts +++ b/src/services/graph/GraphBuilder.ts @@ -90,6 +90,14 @@ export class GraphBuilder { const nodes: Array<{ data: GraphNode }> = []; for (const note of notes) { const degree = degreeMap.get(note.id) ?? 0; + const community = communities.get(note.id); + const size = sizes.get(note.id); + if (community === undefined || size === undefined) { + console.error( + `Note ${note.id} missing from community or size map (expected every note to be covered); defaulting to community 0, size 1.` + ); + } + const label = note.title || '(untitled)'; nodes.push({ data: { @@ -97,8 +105,8 @@ export class GraphBuilder { label: label.length > 64 ? label.substring(0, 61) + '...' : label, noteId: note.id, degree, - community: communities.get(note.id) ?? 0, - size: sizes.get(note.id) ?? 1, + community: community ?? 0, + size: size ?? 1, }, }); } diff --git a/src/services/graph/LouvainDetector.test.ts b/src/services/graph/LouvainDetector.test.ts index bb69c44..e7c21fb 100644 --- a/src/services/graph/LouvainDetector.test.ts +++ b/src/services/graph/LouvainDetector.test.ts @@ -36,17 +36,17 @@ describe('LouvainDetector', () => { expect(communities.get('a')).not.toBe(communities.get('c')); }); - it('falls back to keyword grouping when there are fewer than 3 notes, even with edges', () => { + it('keeps directly linked notes together even under the 3-note Louvain threshold, despite differing keywords', () => { const notes = [note('a', 'Alpha document'), note('b', 'Beta document')]; const edges: GraphEdge[] = [{ source: 'a', target: 'b', type: 'link' }]; const communities = detector.detectCommunities(notes, edges); expect(communities.size).toBe(2); - expect(communities.get('a')).not.toBe(communities.get('b')); + expect(communities.get('a')).toBe(communities.get('b')); }); - it('gives each note its own community when no keyword repeats across notes', () => { + it('gives each note its own community when no keyword repeats and nothing links them', () => { const notes = [ note('a', 'Zebra migration'), note('b', 'Quantum entanglement'), @@ -55,7 +55,7 @@ describe('LouvainDetector', () => { const communities = detector.detectCommunities(notes, []); - expect(new Set(communities.values()).size).toBe(3); + expect(new Set(communities.values())).toEqual(new Set([0, 1, 2])); }); it('numbers keyword-fallback communities by descending size, not by first-appearance order', () => { @@ -77,6 +77,15 @@ describe('LouvainDetector', () => { expect(communities.get('solo1')).not.toBe(0); expect(communities.get('solo2')).not.toBe(0); }); + + it('logs when the graph is too sparse for Louvain and the keyword/link fallback is used', () => { + const consoleInfoSpy = jest.spyOn(console, 'info').mockImplementation(() => undefined); + + detector.detectCommunities([note('a', 'Alpha'), note('b', 'Beta')], []); + + expect(consoleInfoSpy).toHaveBeenCalledWith(expect.stringContaining('too sparse for Louvain')); + consoleInfoSpy.mockRestore(); + }); }); describe('Louvain', () => { @@ -134,6 +143,41 @@ describe('LouvainDetector', () => { expect(Array.from(second.entries())).toEqual(Array.from(first.entries())); }); + it('produces identical assignments regardless of the order notes and edges are supplied in', () => { + // Two triangles bridged by a single edge each to node x - a genuine modularity tie, + // since x has no reason to prefer one triangle over the other. Only the order notes/edges + // arrive in should be able to break the tie one way or the other; that order must not + // leak in from the caller (e.g. a note-fetch order that isn't guaranteed stable). + // Uses fixed shuffles rather than a plain .reverse() - a reversal of this symmetric + // fixture can coincidentally land on the same tie-break, masking the bug this guards. + const triangle = (prefix: string): GraphEdge[] => [ + { source: `${prefix}1`, target: `${prefix}2`, type: 'link' }, + { source: `${prefix}2`, target: `${prefix}3`, type: 'link' }, + { source: `${prefix}1`, target: `${prefix}3`, type: 'link' }, + ]; + const notes = ['x', 'a1', 'a2', 'a3', 'b1', 'b2', 'b3'].map((id) => note(id, id)); + const edges: GraphEdge[] = [ + ...triangle('a'), + ...triangle('b'), + { source: 'x', target: 'a1', type: 'link' }, + { source: 'x', target: 'b1', type: 'link' }, + ]; + + const permute = (arr: T[], order: number[]): T[] => order.map((i) => arr[i]); + const shuffledOrders: Array<{ notes: number[]; edges: number[] }> = [ + { notes: [3, 6, 1, 4, 0, 5, 2], edges: [5, 2, 7, 0, 4, 1, 6, 3] }, + { notes: [6, 5, 4, 3, 2, 1, 0], edges: [7, 6, 5, 4, 3, 2, 1, 0] }, + { notes: [0, 4, 1, 5, 2, 6, 3], edges: [1, 0, 3, 2, 5, 4, 7, 6] }, + ]; + + const expected = Array.from(detector.detectCommunities(notes, edges).entries()).sort(); + + for (const order of shuffledOrders) { + const result = detector.detectCommunities(permute(notes, order.notes), permute(edges, order.edges)); + expect(Array.from(result.entries()).sort()).toEqual(expected); + } + }); + it('weighs a note more strongly toward a cluster it shares multiple edge types with', () => { // x has two relationships with a1 (link + tag) but only one with b1. // Verified against the real library that this specific setup is what @@ -184,7 +228,7 @@ describe('LouvainDetector', () => { expect(() => detector.detectCommunities(notes, edges)).not.toThrow(); }); - it('falls back to keyword grouping when Louvain resolves to near-all singletons', () => { + it('falls back to keyword/link grouping when it consolidates better than a degenerate Louvain result', () => { const notes = [ note('a', 'Linked one'), note('b', 'Linked two'), @@ -197,12 +241,66 @@ describe('LouvainDetector', () => { note('e5', 'Isolate five'), note('e6', 'Isolate six'), ]; - // One edge among 10 otherwise disconnected notes: 9 communities, past the degenerate threshold. + // One edge among 10 otherwise disconnected notes: 9 communities, past the degenerate + // threshold. The keyword fallback consolidates far better here (shared "linked", + // "gardening" and "isolate" keywords), so it must win over the degenerate Louvain result - + // and, since it's the winning path, must still come out size-ordered (id 0 largest). const edges: GraphEdge[] = [{ source: 'a', target: 'b', type: 'link' }]; const communities = detector.detectCommunities(notes, edges); + expect(new Set(communities.values()).size).toBeLessThan(9); expect(communities.get('c')).toBe(communities.get('d')); + expect(communities.get('a')).toBe(communities.get('b')); + + const isolateGroupId = communities.get('e1'); + for (const id of ['e2', 'e3', 'e4', 'e5', 'e6']) { + expect(communities.get(id)).toBe(isolateGroupId); + } + // The 6-note isolate group is the largest community, so it must be id 0. + expect(isolateGroupId).toBe(0); + }); + + it('keeps the degenerate Louvain result when the keyword/link fallback would not consolidate any better', () => { + // Every note has a unique keyword and no two notes share more than one edge, so the + // keyword/link fallback can only merge exactly the pairs already directly linked - + // no better than what Louvain itself found. Falling back here would be a lateral move, + // not an improvement, so the (degenerate) Louvain result should be kept. + const ids = ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j']; + const uniqueTopics = [ + 'Aardvark', + 'Butterfly', + 'Crocodile', + 'Dolphin', + 'Elephant', + 'Flamingo', + 'Giraffe', + 'Hedgehog', + 'Iguana', + 'Jellyfish', + ]; + const notes = ids.map((id, i) => note(id, uniqueTopics[i])); + const edges: GraphEdge[] = [{ source: 'a', target: 'b', type: 'link' }]; + + const communities = detector.detectCommunities(notes, edges); + + expect(communities.get('a')).toBe(communities.get('b')); + expect(new Set(communities.values()).size).toBe(9); + }); + + it('keeps the degenerate Louvain result rather than a keyword/link fallback that collapses almost everyone into one bucket', () => { + // A realistic collapse trigger: templated titles ("Daily Log Entry N") share the same + // dominant keyword across the whole vault, since the fallback can't tell a meaningful + // recurring topic from an incidental template. Falling back here would trade a + // too-fragmented Louvain result for a too-collapsed one - neither is an improvement, + // so the degenerate Louvain result should be kept. + const notes = Array.from({ length: 10 }, (_, i) => note(`d${i}`, `Daily Log Entry ${i}`)); + const edges: GraphEdge[] = [{ source: 'd0', target: 'd1', type: 'link' }]; + + const communities = detector.detectCommunities(notes, edges); + + expect(communities.get('d0')).toBe(communities.get('d1')); + expect(new Set(communities.values()).size).toBe(9); }); }); }); diff --git a/src/services/graph/LouvainDetector.ts b/src/services/graph/LouvainDetector.ts index 566d9a4..a25bdee 100644 --- a/src/services/graph/LouvainDetector.ts +++ b/src/services/graph/LouvainDetector.ts @@ -9,6 +9,9 @@ const MIN_NOTES_FOR_LOUVAIN = 3; /** At or above this ratio of communities to notes, Louvain has basically found nothing (near-all singletons). */ const DEGENERATE_COMMUNITY_RATIO = 0.8; +/** If a single keyword/link-fallback community would hold at least this share of all notes, treat the fallback as a collapse - "everyone lumped into one bucket" is no more meaningful than "everyone in their own bucket". */ +const MAX_FALLBACK_DOMINANT_SHARE = 0.8; + /** Seeded PRNG so the same graph always produces the same Louvain result, instead of the library's default `Math.random` reshuffling colors on every rebuild. */ const createDeterministicRng = (): (() => number) => { let state = 0x9e3779b9; @@ -20,6 +23,9 @@ const createDeterministicRng = (): (() => number) => { }; }; +/** Sorts by note/edge identity so a graph's structure depends only on which notes and edges it contains, never on the order the caller happened to hand them in. */ +const compareStrings = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0); + const STOPWORDS = new Set([ 'this', 'that', @@ -69,25 +75,75 @@ const STOPWORDS = new Set([ 'same', ]); +/** Tracks which notes have been merged into the same community, used to combine keyword grouping with edge connectivity. */ +class DisjointSet { + private readonly parent = new Map(); + + public add(id: string): void { + if (!this.parent.has(id)) { + this.parent.set(id, id); + } + } + + public has(id: string): boolean { + return this.parent.has(id); + } + + public union(a: string, b: string): void { + const rootA = this.find(a); + const rootB = this.find(b); + if (rootA !== rootB) { + this.parent.set(rootA, rootB); + } + } + + public find(id: string): string { + let root = id; + while (this.parent.get(root) !== root) { + root = this.parent.get(root) as string; + } + let current = id; + while (current !== root) { + const next = this.parent.get(current) as string; + this.parent.set(current, root); + current = next; + } + return root; + } +} + /** * Assigns each note to a community. Runs Louvain clustering on the note * graph when it's dense enough to give a meaningful result, and falls back - * to grouping notes by their most frequent keyword otherwise. + * to grouping notes by shared keyword and direct connections otherwise. + * + * Community ids are deterministic and size-ordered (id 0 is always the + * largest community) for a *fixed* note/edge set, so re-running on an + * unchanged graph never reshuffles colors. That guarantee does not extend + * across rebuilds where the corpus itself changes: adding or removing notes + * can change relative community sizes and therefore reassign ids. Anchoring + * ids to a previous rebuild (so unrelated communities don't change color + * when the vault grows) is left for the incremental-update work. */ export class LouvainDetector { - /** - * Both the Louvain and keyword-fallback paths are renumbered by size before returning, so - * callers (e.g. the community color palette) can always rely on id 0 being the largest - * community regardless of which path produced the result. - */ public detectCommunities(notes: Note[], edges: GraphEdge[]): Map { if (this.isTooSparse(notes, edges)) { - return this.renumberBySize(this.groupByKeyword(notes)); + console.info( + `Community detection: graph too sparse for Louvain (${notes.length} notes, ${edges.length} edges), using keyword/link fallback.` + ); + return this.renumberBySize(this.groupByKeyword(notes, edges)); + } + + let raw: Record; + try { + raw = this.runLouvain(notes, edges); + } catch (error) { + console.error('Louvain community detection failed, using keyword/link fallback instead:', error); + return this.renumberBySize(this.groupByKeyword(notes, edges)); } - const raw = this.runLouvain(notes, edges); if (this.isDegenerate(raw, notes.length)) { - return this.renumberBySize(this.groupByKeyword(notes)); + return this.chooseLessFragmented(raw, notes, edges); } return this.renumberBySize(new Map(Object.entries(raw))); } @@ -102,18 +158,60 @@ export class LouvainDetector { return communityCount >= noteCount * DEGENERATE_COMMUNITY_RATIO; } + private chooseLessFragmented( + raw: Record, + notes: Note[], + edges: GraphEdge[] + ): Map { + const louvainResult = new Map(Object.entries(raw)); + const fallbackResult = this.groupByKeyword(notes, edges); + + const louvainCommunityCount = new Set(louvainResult.values()).size; + const fallbackCommunityCount = new Set(fallbackResult.values()).size; + const fallbackIsBetter = + fallbackCommunityCount < louvainCommunityCount && !this.isCollapsed(fallbackResult, notes.length); + + if (fallbackIsBetter) { + console.info( + `Community detection: Louvain result too fragmented (${louvainCommunityCount} communities ` + + `for ${notes.length} notes), using keyword/link fallback (${fallbackCommunityCount} communities).` + ); + return this.renumberBySize(fallbackResult); + } + return this.renumberBySize(louvainResult); + } + + /** True when one community absorbed most of the notes - as meaningless a partition as near-all singletons. */ + private isCollapsed(assignments: Map, noteCount: number): boolean { + const sizeByGroup = new Map(); + for (const groupId of assignments.values()) { + sizeByGroup.set(groupId, (sizeByGroup.get(groupId) ?? 0) + 1); + } + let largestGroupSize = 0; + for (const size of sizeByGroup.values()) { + if (size > largestGroupSize) largestGroupSize = size; + } + return largestGroupSize >= noteCount * MAX_FALLBACK_DOMINANT_SHARE; + } + /** Weights each edge by how many relationships connect the same pair of notes, so a note linked and tagged and semantically similar to another counts for more than a single coincidental edge. */ private runLouvain(notes: Note[], edges: GraphEdge[]): Record { const graph = new Graph({ type: 'undirected' }); - for (const note of notes) { + + const sortedNotes = [...notes].sort((a, b) => compareStrings(a.id, b.id)); + for (const note of sortedNotes) { graph.addNode(note.id); } - for (const edge of edges) { + + const sortedEdges = [...edges].sort( + (a, b) => compareStrings(a.source, b.source) || compareStrings(a.target, b.target) + ); + for (const edge of sortedEdges) { if (!graph.hasNode(edge.source) || !graph.hasNode(edge.target)) { continue; } if (graph.hasEdge(edge.source, edge.target)) { - graph.updateEdgeAttribute(edge.source, edge.target, 'weight', (w) => (w ?? 1) + 1); + graph.updateEdgeAttribute(edge.source, edge.target, 'weight', (w) => w + 1); } else { graph.mergeEdge(edge.source, edge.target, { weight: 1 }); } @@ -138,7 +236,7 @@ export class LouvainDetector { members, minId: members.reduce((min, id) => (id < min ? id : min)), })); - groups.sort((a, b) => b.members.length - a.members.length || (a.minId < b.minId ? -1 : 1)); + groups.sort((a, b) => b.members.length - a.members.length || compareStrings(a.minId, b.minId)); const renumbered = new Map(); groups.forEach(({ members }, newId) => { @@ -149,20 +247,40 @@ export class LouvainDetector { return renumbered; } - private groupByKeyword(notes: Note[]): Map { - const communityByKeyword = new Map(); - const assignments = new Map(); + private groupByKeyword(notes: Note[], edges: GraphEdge[]): Map { + const groups = new DisjointSet(); + for (const note of notes) { + groups.add(note.id); + } + const representativeByKeyword = new Map(); for (const note of notes) { const keyword = this.extractKeyword(note); - let community = communityByKeyword.get(keyword); - if (community === undefined) { - community = communityByKeyword.size; - communityByKeyword.set(keyword, community); + const representative = representativeByKeyword.get(keyword); + if (representative) { + groups.union(note.id, representative); + } else { + representativeByKeyword.set(keyword, note.id); + } + } + + for (const edge of edges) { + if (groups.has(edge.source) && groups.has(edge.target)) { + groups.union(edge.source, edge.target); } - assignments.set(note.id, community); } + const assignments = new Map(); + const idByRoot = new Map(); + for (const note of notes) { + const root = groups.find(note.id); + let id = idByRoot.get(root); + if (id === undefined) { + id = idByRoot.size; + idByRoot.set(root, id); + } + assignments.set(note.id, id); + } return assignments; } From 8f8b2ff6667531dc4e0a11d6f8cfb9b4076a3f89 Mon Sep 17 00:00:00 2001 From: Yugal Kaushik Date: Tue, 4 Aug 2026 23:11:38 +0530 Subject: [PATCH 28/28] dev: code improvements based on comments --- src/services/graph/LouvainDetector.ts | 6 ++++- src/services/similarity/EdgeFactory.test.ts | 11 +++++---- src/services/similarity/EdgeFactory.ts | 25 +++++++++++++-------- src/services/similarity/SimilarityEngine.ts | 11 ++++++--- 4 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/services/graph/LouvainDetector.ts b/src/services/graph/LouvainDetector.ts index a25bdee..c5ed40d 100644 --- a/src/services/graph/LouvainDetector.ts +++ b/src/services/graph/LouvainDetector.ts @@ -304,7 +304,11 @@ export class LouvainDetector { return bestWord ?? `note:${note.id}`; } - /** Latin-script words only; other scripts fall through to the per-note key above. */ + /** + * Latin-script words only; other scripts fall through to the per-note key above. + * TODO: extend the regex (or use a script-aware tokenizer) to support non-Latin + * scripts as a post-GSoC enhancement. + */ private tokenize(text: string): string[] { return text.toLowerCase().match(/[a-z]{2,}/g) ?? []; } diff --git a/src/services/similarity/EdgeFactory.test.ts b/src/services/similarity/EdgeFactory.test.ts index 61ef6ee..4a6f144 100644 --- a/src/services/similarity/EdgeFactory.test.ts +++ b/src/services/similarity/EdgeFactory.test.ts @@ -39,11 +39,14 @@ describe('EdgeFactory', () => { expect(edges).toEqual([{ source: 'a', target: 'b', type: 'link' }]); }); - it('creates bidirectional links when notes reference each other', () => { + it('collapses a mutual link into a single edge', () => { const edges = factory.createEdges([note('a', 'A', ['b']), note('b', 'B', ['a'])]); - expect(edges).toHaveLength(2); - expect(edges).toContainEqual({ source: 'a', target: 'b', type: 'link' }); - expect(edges).toContainEqual({ source: 'b', target: 'a', type: 'link' }); + expect(edges).toEqual([{ source: 'a', target: 'b', type: 'link' }]); + }); + + it('normalizes link edges to id order regardless of authored direction', () => { + const edges = factory.createEdges([note('z', 'Z', ['a']), note('a', 'A', [])]); + expect(edges).toEqual([{ source: 'a', target: 'z', type: 'link' }]); }); it('creates tag edge with tagName for shared tags', () => { diff --git a/src/services/similarity/EdgeFactory.ts b/src/services/similarity/EdgeFactory.ts index 0197272..346cf62 100644 --- a/src/services/similarity/EdgeFactory.ts +++ b/src/services/similarity/EdgeFactory.ts @@ -2,6 +2,9 @@ import { Note } from '../../data/Types'; import { GraphEdge } from '../graph/types'; import { SimilarityPair } from './SimilarityEngine'; +/** Tags shared by more notes than this are skipped entirely, to avoid a combinatorial blowup of pairs (a clique on n notes is n*(n-1)/2 edges). */ +const MAX_NOTES_PER_TAG = 20; + export class EdgeFactory { /** * Creates graph edges from explicit note links and shared tags. @@ -12,25 +15,29 @@ export class EdgeFactory { return [...this.createLinkEdges(notes), ...this.createTagEdges(notes)]; } - /** Builds one deduplicated edge per explicit `:/noteId` link between two notes in scope. */ + /** + * Builds one deduplicated edge per explicit `:/noteId` link between two notes + * in scope. Direction-agnostic, same as `createTagEdges`: a mutual A<->B link + * is one edge, not two, and its `source`/`target` are normalized to id order + * rather than kept in authored order. + */ private createLinkEdges(notes: Note[]): GraphEdge[] { const noteIdSet = new Set(notes.map((n) => n.id)); - const edges: GraphEdge[] = []; - const linkKeySet = new Set(); + const linkEdgeMap = new Map(); for (const note of notes) { for (const link of note.links ?? []) { if (noteIdSet.has(link) && link !== note.id) { - const key = `${note.id}::${link}::link`; - if (!linkKeySet.has(key)) { - linkKeySet.add(key); - edges.push({ source: note.id, target: link, type: 'link' }); + const [a, b] = note.id < link ? [note.id, link] : [link, note.id]; + const pairKey = `${a}::${b}`; + if (!linkEdgeMap.has(pairKey)) { + linkEdgeMap.set(pairKey, { source: a, target: b, type: 'link' }); } } } } - return edges; + return Array.from(linkEdgeMap.values()); } /** @@ -43,7 +50,7 @@ export class EdgeFactory { const tagEdgeMap = new Map(); for (const [tagName, noteIds] of tagToNotes) { - if (noteIds.length > 20) continue; + if (noteIds.length > MAX_NOTES_PER_TAG) continue; for (let i = 0; i < noteIds.length; i++) { for (let j = i + 1; j < noteIds.length; j++) { diff --git a/src/services/similarity/SimilarityEngine.ts b/src/services/similarity/SimilarityEngine.ts index b879152..1bbb256 100644 --- a/src/services/similarity/SimilarityEngine.ts +++ b/src/services/similarity/SimilarityEngine.ts @@ -70,7 +70,7 @@ export class SimilarityEngine { return []; } - const normalized = this.normalize(aboveFloor); + const normalized = this.normalize(aboveFloor, SEMANTIC_FLOOR); const enriched = this.addBonusPoints(normalized); const aboveThreshold = this.filterBelowThreshold(enriched, threshold); const topPairs = this.selectTopK(aboveThreshold, topK); @@ -194,16 +194,21 @@ export class SimilarityEngine { return sum; } - /** Min-max normalizes scores to [0, 1]. Skips if spread is too narrow. */ - private normalize(pairs: SimilarityPair[]): SimilarityPair[] { + /** Min-max normalizes scores to [0, 1], using only pairs that clear `floor` on their own to compute the range (excludes link-kept sub-floor outliers, see `filterBelowFloor`). */ + private normalize(pairs: SimilarityPair[], floor: number): SimilarityPair[] { let min = Infinity; let max = -Infinity; for (const p of pairs) { + if (p.score < floor) continue; if (p.score < min) min = p.score; if (p.score > max) max = p.score; } + if (min === Infinity) { + return pairs; + } + const spread = max - min; if (spread < 0.1) { return pairs;