diff --git a/src/data/Database/GraphCacheRepository.test.ts b/src/data/Database/GraphCacheRepository.test.ts new file mode 100644 index 0000000..3680f97 --- /dev/null +++ b/src/data/Database/GraphCacheRepository.test.ts @@ -0,0 +1,155 @@ +import { GraphCacheRepository } from './GraphCacheRepository'; +import { IVectorDatabase } from './VectorDatabase'; +import { GraphData } from '../../services/graph/types'; +import { Note } from '../Types'; + +class FakeConnection implements IVectorDatabase { + public opened = false; + private graphRow: { notes_json: string; graph_json: string } | null = null; + private syncStateRow: { events_cursor: string | null; embeddings_cursor: string | null } | null = + null; + + public async open(): Promise { + this.opened = true; + } + + public async run(sql: string, params: unknown[]): Promise { + if (sql.includes('INTO graph_cache')) { + const [notesJson, graphJson] = params as [string, string, number]; + this.graphRow = { notes_json: notesJson, graph_json: graphJson }; + } else if (sql.includes('embeddings_cursor')) { + const [cursor] = params as [string]; + this.syncStateRow = { + events_cursor: this.syncStateRow?.events_cursor ?? null, + embeddings_cursor: cursor, + }; + } else if (sql.includes('INTO sync_state')) { + const [cursor] = params as [string]; + this.syncStateRow = { + events_cursor: cursor, + embeddings_cursor: this.syncStateRow?.embeddings_cursor ?? null, + }; + } + } + + public async all(sql: string): Promise { + if (sql.includes('FROM graph_cache')) { + return (this.graphRow ? [this.graphRow] : []) as unknown as T[]; + } + if (sql.includes('FROM sync_state')) { + return (this.syncStateRow ? [this.syncStateRow] : []) as unknown as T[]; + } + return []; + } +} + +const note: Note = { + id: 'n1', + parent_id: 'p1', + title: 'Note 1', + body: 'body', + created_time: 1, + updated_time: 2, +}; + +const graphData: GraphData = { + nodes: [{ data: { id: 'n1', label: 'Note 1', noteId: 'n1', degree: 0, community: 0, size: 1 } }], + edges: [], +}; + +describe('GraphCacheRepository', () => { + let db: FakeConnection; + let repo: GraphCacheRepository; + + beforeEach(() => { + db = new FakeConnection(); + repo = new GraphCacheRepository(db); + }); + + describe('graph cache', () => { + it('returns null when nothing has been cached yet', async () => { + const result = await repo.loadGraph(); + expect(result).toBeNull(); + }); + + it('round-trips notes and graph data through save/load', async () => { + await repo.saveGraph([note], graphData); + + const result = await repo.loadGraph(); + + expect(result).toEqual({ notes: [note], graphData }); + }); + + it('overwrites the previous cache on a second save', async () => { + await repo.saveGraph([note], graphData); + const secondNote = { ...note, title: 'Updated' }; + await repo.saveGraph([secondNote], graphData); + + const result = await repo.loadGraph(); + + expect(result?.notes[0].title).toBe('Updated'); + }); + }); + + describe('events cursor', () => { + it('returns null when sync has never run', async () => { + const cursor = await repo.loadEventsCursor(); + expect(cursor).toBeNull(); + }); + + it('round-trips the cursor through save/load', async () => { + await repo.saveEventsCursor('cursor-1'); + expect(await repo.loadEventsCursor()).toBe('cursor-1'); + }); + + it('overwrites the previous cursor on a second save', async () => { + await repo.saveEventsCursor('cursor-1'); + await repo.saveEventsCursor('cursor-2'); + expect(await repo.loadEventsCursor()).toBe('cursor-2'); + }); + }); + + describe('embeddings cursor', () => { + it('returns null when the AI-on sweep has never run', async () => { + const cursor = await repo.loadEmbeddingsCursor(); + expect(cursor).toBeNull(); + }); + + it('round-trips the cursor through save/load', async () => { + await repo.saveEmbeddingsCursor('embeddings-cursor-1'); + expect(await repo.loadEmbeddingsCursor()).toBe('embeddings-cursor-1'); + }); + + it('overwrites the previous cursor on a second save', async () => { + await repo.saveEmbeddingsCursor('embeddings-cursor-1'); + await repo.saveEmbeddingsCursor('embeddings-cursor-2'); + expect(await repo.loadEmbeddingsCursor()).toBe('embeddings-cursor-2'); + }); + + it('saving the events cursor and the embeddings cursor never clobbers the other', async () => { + await repo.saveEventsCursor('events-cursor-1'); + await repo.saveEmbeddingsCursor('embeddings-cursor-1'); + await repo.saveEventsCursor('events-cursor-2'); + + expect(await repo.loadEventsCursor()).toBe('events-cursor-2'); + expect(await repo.loadEmbeddingsCursor()).toBe('embeddings-cursor-1'); + }); + }); + + describe('write serialization', () => { + it('serializes interleaved graph and cursor writes instead of racing them', async () => { + const order: string[] = []; + const originalRun = db.run.bind(db); + db.run = async (sql: string, params: unknown[]) => { + order.push(sql.includes('graph_cache') ? 'graph' : 'cursor'); + await originalRun(sql, params); + }; + + await Promise.all([repo.saveGraph([note], graphData), repo.saveEventsCursor('cursor-1')]); + + expect(order).toHaveLength(2); + expect(await repo.loadGraph()).not.toBeNull(); + expect(await repo.loadEventsCursor()).toBe('cursor-1'); + }); + }); +}); diff --git a/src/data/Database/GraphCacheRepository.ts b/src/data/Database/GraphCacheRepository.ts new file mode 100644 index 0000000..fb4ba0c --- /dev/null +++ b/src/data/Database/GraphCacheRepository.ts @@ -0,0 +1,124 @@ +import { IVectorDatabase, VectorDatabase } from './VectorDatabase'; +import { Note } from '../Types'; +import { GraphData } from '../../services/graph/types'; + +const DB_FILE_NAME = 'note-graph-cache.sqlite'; + +const GRAPH_CACHE_SCHEMA = ` + CREATE TABLE IF NOT EXISTS graph_cache ( + id INTEGER PRIMARY KEY CHECK (id = 1), + notes_json TEXT NOT NULL, + graph_json TEXT NOT NULL, + updated_time INTEGER NOT NULL + ) +`; + +const SYNC_STATE_SCHEMA = ` + CREATE TABLE IF NOT EXISTS sync_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + events_cursor TEXT, + embeddings_cursor TEXT + ) +`; + +interface GraphCacheRow { + notes_json: string; + graph_json: string; +} + +interface SyncStateRow { + events_cursor: string | null; + embeddings_cursor: string | null; +} + +export class GraphCacheRepository { + private writeLock: Promise = Promise.resolve(); + + public constructor( + private readonly db: IVectorDatabase = new VectorDatabase(DB_FILE_NAME, [ + GRAPH_CACHE_SCHEMA, + SYNC_STATE_SCHEMA, + ]) + ) {} + + public async loadGraph(): Promise<{ notes: Note[]; graphData: GraphData } | null> { + await this.db.open(); + const rows = await this.db.all( + 'SELECT notes_json, graph_json FROM graph_cache WHERE id = 1', + [] + ); + const row = rows[0]; + if (!row) return null; + + return { + notes: JSON.parse(row.notes_json) as Note[], + graphData: JSON.parse(row.graph_json) as GraphData, + }; + } + + public saveGraph(notes: Note[], graphData: GraphData): Promise { + return this.enqueueWrite(async () => { + await this.db.open(); + await this.db.run( + `INSERT INTO graph_cache (id, notes_json, graph_json, updated_time) + VALUES (1, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + notes_json = excluded.notes_json, + graph_json = excluded.graph_json, + updated_time = excluded.updated_time`, + [JSON.stringify(notes), JSON.stringify(graphData), Date.now()] + ); + }); + } + + public async loadEventsCursor(): Promise { + await this.db.open(); + const rows = await this.db.all( + 'SELECT events_cursor FROM sync_state WHERE id = 1', + [] + ); + return rows[0]?.events_cursor ?? null; + } + + public saveEventsCursor(cursor: string): Promise { + return this.enqueueWrite(async () => { + await this.db.open(); + await this.db.run( + `INSERT INTO sync_state (id, events_cursor) + VALUES (1, ?) + ON CONFLICT(id) DO UPDATE SET events_cursor = excluded.events_cursor`, + [cursor] + ); + }); + } + + public async loadEmbeddingsCursor(): Promise { + await this.db.open(); + const rows = await this.db.all( + 'SELECT embeddings_cursor FROM sync_state WHERE id = 1', + [] + ); + return rows[0]?.embeddings_cursor ?? null; + } + + public saveEmbeddingsCursor(cursor: string): Promise { + return this.enqueueWrite(async () => { + await this.db.open(); + await this.db.run( + `INSERT INTO sync_state (id, embeddings_cursor) + VALUES (1, ?) + ON CONFLICT(id) DO UPDATE SET embeddings_cursor = excluded.embeddings_cursor`, + [cursor] + ); + }); + } + + private enqueueWrite(write: () => Promise): Promise { + const task = this.writeLock.then(write); + this.writeLock = task.then( + () => undefined, + () => undefined + ); + return task; + } +} diff --git a/src/data/Database/VectorDatabase.ts b/src/data/Database/VectorDatabase.ts index a3a79b7..156ff29 100644 --- a/src/data/Database/VectorDatabase.ts +++ b/src/data/Database/VectorDatabase.ts @@ -19,30 +19,25 @@ interface Sqlite3Database { /** * Thin promisified wrapper around Joplin's bundled sqlite3 module (accessed via * `joplin.require('sqlite3')`, since native packages can't be bundled with a - * plugin). Owns only the connection and schema; query logic lives in - * VectorRepository. + * plugin). Owns only the connection and schema; query logic lives in the + * repository classes that use it. */ export class VectorDatabase implements IVectorDatabase { - private static readonly DB_FILE_NAME = 'note-graph-vectors.sqlite'; - private static readonly SCHEMA = ` - CREATE TABLE IF NOT EXISTS note_vectors ( - note_id TEXT PRIMARY KEY, - model_id TEXT NOT NULL, - updated_time INTEGER NOT NULL, - vector BLOB NOT NULL - ) - `; - private db: Sqlite3Database | null = null; private opening: Promise | null = null; + public constructor( + private readonly dbFileName: string, + private readonly schemaStatements: string[] + ) {} + /** - * Opens (creating if needed) the vector cache database. Safe to call - * repeatedly. A failed open is not cached: both `opening` and `db` are - * reset on rejection so a later call can retry from scratch, instead of - * either re-awaiting the same stale rejection or (if the connection - * itself succeeded but schema creation failed) treating a half-open - * database as ready forever. + * Opens (creating if needed) the database. Safe to call repeatedly. A + * failed open is not cached: both `opening` and `db` are reset on + * rejection so a later call can retry from scratch, instead of either + * re-awaiting the same stale rejection or (if the connection itself + * succeeded but schema creation failed) treating a half-open database as + * ready forever. */ public async open(): Promise { if (this.db) return; @@ -76,7 +71,7 @@ export class VectorDatabase implements IVectorDatabase { private async openInternal(): Promise { const sqlite3 = joplin.require('sqlite3'); const dataDir = await joplin.plugins.dataDir(); - const dbPath = `${dataDir}/${VectorDatabase.DB_FILE_NAME}`; + const dbPath = `${dataDir}/${this.dbFileName}`; this.db = await new Promise((resolve, reject) => { const db = new sqlite3.Database(dbPath, (err: Error | null) => { @@ -85,12 +80,14 @@ export class VectorDatabase implements IVectorDatabase { }); }); - await this.run(VectorDatabase.SCHEMA, []); + for (const statement of this.schemaStatements) { + await this.run(statement, []); + } } private requireDb(): Sqlite3Database { if (!this.db) { - throw new Error('VectorDatabase used before open() completed.'); + throw new Error(`VectorDatabase (${this.dbFileName}) used before open() completed.`); } return this.db; } diff --git a/src/data/Database/VectorRepository.ts b/src/data/Database/VectorRepository.ts index 55c2127..ac7a5d4 100644 --- a/src/data/Database/VectorRepository.ts +++ b/src/data/Database/VectorRepository.ts @@ -1,5 +1,15 @@ import { IVectorDatabase, VectorDatabase } from './VectorDatabase'; +const DB_FILE_NAME = 'note-graph-vectors.sqlite'; +const SCHEMA = ` + CREATE TABLE IF NOT EXISTS note_vectors ( + note_id TEXT PRIMARY KEY, + model_id TEXT NOT NULL, + updated_time INTEGER NOT NULL, + vector BLOB NOT NULL + ) +`; + export interface CachedVector { vector: number[]; modelId: string; @@ -41,7 +51,9 @@ export class VectorRepository implements VectorCache { */ private writeLock: Promise = Promise.resolve(); - public constructor(private readonly db: IVectorDatabase = new VectorDatabase()) {} + public constructor( + private readonly db: IVectorDatabase = new VectorDatabase(DB_FILE_NAME, [SCHEMA]) + ) {} /** Returns cached vectors for the given note IDs, keyed by note ID. Missing notes are omitted. */ public async getMany(noteIds: string[]): Promise> { diff --git a/src/data/EventsRepository.test.ts b/src/data/EventsRepository.test.ts new file mode 100644 index 0000000..4040b9d --- /dev/null +++ b/src/data/EventsRepository.test.ts @@ -0,0 +1,136 @@ +import { EventsRepository } from './EventsRepository'; +import joplin from 'api'; + +const mockGet = joplin.data.get as jest.Mock; + +describe('EventsRepository', () => { + let repo: EventsRepository; + + beforeEach(() => { + repo = new EventsRepository(); + jest.clearAllMocks(); + }); + + it('returns no items and a baseline cursor on a first-ever call, without sending an undefined cursor', async () => { + mockGet.mockResolvedValueOnce({ items: [], cursor: 'baseline-1', has_more: false }); + + const { events, cursor } = await repo.getNoteEventsSince(); + + expect(events).toEqual([]); + expect(cursor).toBe('baseline-1'); + expect(mockGet).toHaveBeenCalledWith(['events'], {}); + }); + + it('maps created/updated/deleted event codes to readable types', async () => { + mockGet.mockResolvedValueOnce({ + items: [ + { item_type: 1, item_id: 'n1', type: 1 }, + { item_type: 1, item_id: 'n2', type: 2 }, + { item_type: 1, item_id: 'n3', type: 3 }, + ], + cursor: 'c2', + has_more: false, + }); + + const { events } = await repo.getNoteEventsSince('c1'); + + expect(events).toEqual( + expect.arrayContaining([ + { noteId: 'n1', type: 'created' }, + { noteId: 'n2', type: 'updated' }, + { noteId: 'n3', type: 'deleted' }, + ]) + ); + expect(mockGet).toHaveBeenCalledWith(['events'], { cursor: 'c1' }); + }); + + it('ignores events for item types other than notes', async () => { + mockGet.mockResolvedValueOnce({ + items: [ + { item_type: 2, item_id: 'folder1', type: 2 }, + { item_type: 1, item_id: 'n1', type: 2 }, + ], + cursor: 'c2', + has_more: false, + }); + + const { events } = await repo.getNoteEventsSince('c1'); + + expect(events).toEqual([{ noteId: 'n1', type: 'updated' }]); + }); + + it('keeps only the latest event per note across the swept window', async () => { + mockGet.mockResolvedValueOnce({ + items: [ + { item_type: 1, item_id: 'n1', type: 1 }, + { item_type: 1, item_id: 'n1', type: 2 }, + ], + cursor: 'c2', + has_more: false, + }); + + const { events } = await repo.getNoteEventsSince('c1'); + + expect(events).toEqual([{ noteId: 'n1', type: 'updated' }]); + }); + + it('nets a created-then-deleted note to deleted', async () => { + mockGet.mockResolvedValueOnce({ + items: [ + { item_type: 1, item_id: 'n1', type: 1 }, + { item_type: 1, item_id: 'n1', type: 3 }, + ], + cursor: 'c2', + has_more: false, + }); + + const { events } = await repo.getNoteEventsSince('c1'); + + expect(events).toEqual([{ noteId: 'n1', type: 'deleted' }]); + }); + + it('pages through multiple event pages, resuming with each returned cursor', async () => { + mockGet + .mockResolvedValueOnce({ + items: [{ item_type: 1, item_id: 'n1', type: 2 }], + cursor: 'c2', + has_more: true, + }) + .mockResolvedValueOnce({ + items: [{ item_type: 1, item_id: 'n2', type: 1 }], + cursor: 'c3', + has_more: false, + }); + + const { events, cursor } = await repo.getNoteEventsSince('c1'); + + expect(events).toEqual( + expect.arrayContaining([ + { noteId: 'n1', type: 'updated' }, + { noteId: 'n2', type: 'created' }, + ]) + ); + expect(cursor).toBe('c3'); + expect(mockGet).toHaveBeenNthCalledWith(1, ['events'], { cursor: 'c1' }); + expect(mockGet).toHaveBeenNthCalledWith(2, ['events'], { cursor: 'c2' }); + }); + + it('stops at the page safety cap and returns the cursor reached so far', async () => { + mockGet.mockResolvedValue({ + items: [{ item_type: 1, item_id: 'n1', type: 2 }], + cursor: 'still-going', + has_more: true, + }); + + const { cursor } = await repo.getNoteEventsSince('c1'); + + expect(cursor).toBe('still-going'); + expect(mockGet).toHaveBeenCalledTimes(50); + }); + + it('propagates a fetch failure instead of swallowing it', async () => { + mockGet.mockRejectedValueOnce(new Error('network error')); + + await expect(repo.getNoteEventsSince('c1')).rejects.toThrow('network error'); + }); +}); diff --git a/src/data/EventsRepository.ts b/src/data/EventsRepository.ts new file mode 100644 index 0000000..98b0b50 --- /dev/null +++ b/src/data/EventsRepository.ts @@ -0,0 +1,67 @@ +import joplin from 'api'; + +export type NoteChangeType = 'created' | 'updated' | 'deleted'; + +export interface NoteEvent { + noteId: string; + type: NoteChangeType; +} + +const NOTE_ITEM_TYPE = 1; + +const EVENT_TYPE_BY_CODE: Record = { + 1: 'created', + 2: 'updated', + 3: 'deleted', +}; + +interface EventItem { + item_type: number; + item_id: string; + type: number; +} + +interface EventsPage { + items?: EventItem[]; + cursor?: string; + has_more?: boolean; +} + +export class EventsRepository { + private static readonly MAX_PAGES = 50; + + public async getNoteEventsSince( + cursor?: string + ): Promise<{ events: NoteEvent[]; cursor: string | undefined }> { + const latestByNoteId = new Map(); + let currentCursor = cursor; + let pageCount = 0; + + while (pageCount < EventsRepository.MAX_PAGES) { + pageCount++; + const query = currentCursor ? { cursor: currentCursor } : {}; + const response: EventsPage = await joplin.data.get(['events'], query); + + for (const item of response.items ?? []) { + if (item.item_type !== NOTE_ITEM_TYPE) continue; + const type = EVENT_TYPE_BY_CODE[item.type]; + if (!type) continue; + latestByNoteId.set(item.item_id, type); + } + + currentCursor = response.cursor; + if (response.has_more !== true) break; + } + + if (pageCount >= EventsRepository.MAX_PAGES) { + console.info( + `Events sweep hit the ${EventsRepository.MAX_PAGES}-page safety cap; remaining events will be picked up on the next sync.` + ); + } + + return { + events: Array.from(latestByNoteId, ([noteId, type]) => ({ noteId, type })), + cursor: currentCursor, + }; + } +} diff --git a/src/data/NotePreprocessor.test.ts b/src/data/NotePreprocessor.test.ts index 62a5209..28520b3 100644 --- a/src/data/NotePreprocessor.test.ts +++ b/src/data/NotePreprocessor.test.ts @@ -60,6 +60,31 @@ describe('NotePreprocessor', () => { expect(mockTagRepositoryInstance.getNoteTagsMap).toHaveBeenCalled(); }); + it('logs, but still returns notes, when the bulk tag fetch was truncated', async () => { + const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => undefined); + mockLinkExtractorInstance.extractLinks.mockReturnValue([]); + mockTagRepositoryInstance.getNoteTagsMap.mockResolvedValue({ map: {}, truncated: true }); + + const notes = [ + { + id: 'n1', + parent_id: 'p1', + title: 'Test', + body: '', + created_time: 0, + updated_time: 1, + }, + ]; + + const result = await preprocessor.process(notes); + + expect(result).toHaveLength(1); + expect(consoleErrorSpy).toHaveBeenCalledWith( + expect.stringContaining('Tag data is incomplete') + ); + consoleErrorSpy.mockRestore(); + }); + it('handles notes with no links and no tags', async () => { mockLinkExtractorInstance.extractLinks.mockReturnValue([]); @@ -107,4 +132,48 @@ describe('NotePreprocessor', () => { expect(mockLinkExtractorInstance.extractLinks).toHaveBeenCalledWith(''); }); + + describe('processOne', () => { + it('enriches a single note using a scoped tag lookup, not the full map', async () => { + mockLinkExtractorInstance.extractLinks.mockReturnValue(['abc']); + mockTagRepositoryInstance.getTagsForNote.mockResolvedValue({ + titles: ['tag1'], + truncated: false, + }); + + const note = { + id: 'note1', + parent_id: 'p1', + title: 'Test', + body: 'body :/abc', + created_time: 0, + updated_time: 1, + }; + + const result = await preprocessor.processOne(note); + + expect(result).toMatchObject({ id: 'note1', links: ['abc'], tags: ['tag1'] }); + expect(mockTagRepositoryInstance.getTagsForNote).toHaveBeenCalledWith('note1'); + expect(mockTagRepositoryInstance.getNoteTagsMap).not.toHaveBeenCalled(); + }); + + it('throws instead of silently committing a truncated tag list', async () => { + mockLinkExtractorInstance.extractLinks.mockReturnValue([]); + mockTagRepositoryInstance.getTagsForNote.mockResolvedValue({ + titles: ['tag1'], + truncated: true, + }); + + const note = { + id: 'note1', + parent_id: 'p1', + title: 'Test', + body: '', + created_time: 0, + updated_time: 1, + }; + + await expect(preprocessor.processOne(note)).rejects.toThrow(/note1/); + }); + }); }); diff --git a/src/data/NotePreprocessor.ts b/src/data/NotePreprocessor.ts index bce5246..5d3f7a3 100644 --- a/src/data/NotePreprocessor.ts +++ b/src/data/NotePreprocessor.ts @@ -17,7 +17,10 @@ export class NotePreprocessor { * @returns the same notes with `links` and `tags` populated. */ public async process(notes: Note[]): Promise { - const { map: noteTagsMap } = await this.tagRepository.getNoteTagsMap(); + const { map: noteTagsMap, truncated } = await this.tagRepository.getNoteTagsMap(); + if (truncated) { + console.error('Tag data is incomplete for this reload - some tag connections may be missing.'); + } return notes.map((note) => ({ ...note, @@ -25,4 +28,16 @@ export class NotePreprocessor { tags: noteTagsMap[note.id] ?? [], })); } + + public async processOne(note: Note): Promise { + const { titles, truncated } = await this.tagRepository.getTagsForNote(note.id); + if (truncated) { + throw new Error(`Could not fetch the complete tag list for note ${note.id}.`); + } + return { + ...note, + links: this.linkExtractor.extractLinks(note.body ?? ''), + tags: titles, + }; + } } diff --git a/src/data/NoteRepository.test.ts b/src/data/NoteRepository.test.ts index c7f33f4..d05d5b1 100644 --- a/src/data/NoteRepository.test.ts +++ b/src/data/NoteRepository.test.ts @@ -75,12 +75,27 @@ describe('NoteRepository', () => { await repo.getAllNotes(); expect(mockGet).toHaveBeenCalledWith(['notes'], { - fields: ['id', 'parent_id', 'title', 'body', 'created_time', 'updated_time'], + fields: ['id', 'parent_id', 'title', 'body', 'created_time', 'updated_time', 'deleted_time'], limit: 100, page: 1, }); }); + it('filters out notes that are in the trash', async () => { + mockGet.mockResolvedValueOnce({ + items: [ + { id: '1', deleted_time: 0 }, + { id: '2', deleted_time: 1700000000000 }, + { id: '3', deleted_time: 0 }, + ], + has_more: false, + }); + + const { notes } = await repo.getAllNotes(); + + expect(notes.map((n) => n.id)).toEqual(['1', '3']); + }); + it('handles missing items in response gracefully', async () => { mockGet.mockResolvedValueOnce({ has_more: false, @@ -145,4 +160,47 @@ describe('NoteRepository', () => { expect(notes).toHaveLength(2); expect(mockGet).toHaveBeenCalledTimes(1); }); + + describe('getNote', () => { + it('fetches a single note by ID with the standard fields', async () => { + mockGet.mockResolvedValueOnce({ + id: '1', + parent_id: 'p1', + title: 'Note 1', + body: 'Body', + created_time: 100, + updated_time: 200, + deleted_time: 0, + }); + + const note = await repo.getNote('1'); + + expect(note).toMatchObject({ id: '1', title: 'Note 1' }); + expect(mockGet).toHaveBeenCalledWith(['notes', '1'], { + fields: ['id', 'parent_id', 'title', 'body', 'created_time', 'updated_time', 'deleted_time'], + }); + }); + + it('returns null when the note no longer exists', async () => { + mockGet.mockRejectedValueOnce(new Error('Not Found')); + + const note = await repo.getNote('missing'); + + expect(note).toBeNull(); + }); + + it('returns null when the note has been moved to the trash', async () => { + mockGet.mockResolvedValueOnce({ id: '1', deleted_time: 1700000000000 }); + + const note = await repo.getNote('1'); + + expect(note).toBeNull(); + }); + + it('rethrows when the fetch fails for a reason other than the note being deleted', async () => { + mockGet.mockRejectedValueOnce(new Error('network error')); + + await expect(repo.getNote('1')).rejects.toThrow('network error'); + }); + }); }); diff --git a/src/data/NoteRepository.ts b/src/data/NoteRepository.ts index fd6c3e4..b770609 100644 --- a/src/data/NoteRepository.ts +++ b/src/data/NoteRepository.ts @@ -1,6 +1,12 @@ import joplin from 'api'; import { Note } from './Types'; +const NOTE_FIELDS = ['id', 'parent_id', 'title', 'body', 'created_time', 'updated_time', 'deleted_time']; + +interface NoteResponse extends Note { + deleted_time: number; +} + export class NoteRepository { /** * Fetches all notes from the Joplin API with pagination. @@ -19,12 +25,13 @@ export class NoteRepository { try { const response = await joplin.data.get(['notes'], { - fields: ['id', 'parent_id', 'title', 'body', 'created_time', 'updated_time'], + fields: NOTE_FIELDS, limit: Math.min(remaining, 100), page, }); - const items = (response.items ?? []).slice(0, remaining); - notes.push(...items); + const items: NoteResponse[] = response.items ?? []; + const active = items.filter((n) => !n.deleted_time).slice(0, remaining); + notes.push(...active); hasMore = response.has_more === true; page++; } catch (error) { @@ -35,4 +42,24 @@ export class NoteRepository { console.info(`Fetched ${notes.length} notes.`); return { notes, truncated: false }; } + + public async getNote(id: string): Promise { + try { + const note: NoteResponse = await joplin.data.get(['notes', id], { + fields: NOTE_FIELDS, + }); + if (note.deleted_time) { + console.info(`Note ${id} is in the trash.`); + return null; + } + return note; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (!message.includes('Not Found')) { + throw error; + } + console.info(`Note ${id} no longer exists.`); + return null; + } + } } diff --git a/src/data/TagRepository.test.ts b/src/data/TagRepository.test.ts index 95183ee..965fb4c 100644 --- a/src/data/TagRepository.test.ts +++ b/src/data/TagRepository.test.ts @@ -170,4 +170,66 @@ describe('TagRepository', () => { }); expect(mockGet).toHaveBeenCalledTimes(3); }); + + describe('getTagsForNote', () => { + it('returns tag titles for a single note without walking all tags', async () => { + mockGet.mockResolvedValueOnce({ + items: [{ title: 'a' }, { title: 'b' }], + has_more: false, + }); + + const { titles, truncated } = await repo.getTagsForNote('note1'); + + expect(titles).toEqual(['a', 'b']); + expect(truncated).toBe(false); + expect(mockGet).toHaveBeenCalledTimes(1); + expect(mockGet).toHaveBeenCalledWith(['notes', 'note1', 'tags'], { + fields: ['title'], + page: 1, + limit: 100, + }); + }); + + it('paginates through a note with many tags', async () => { + mockGet + .mockResolvedValueOnce({ items: [{ title: 'a' }], has_more: true }) + .mockResolvedValueOnce({ items: [{ title: 'b' }], has_more: false }); + + const { titles, truncated } = await repo.getTagsForNote('note1'); + + expect(titles).toEqual(['a', 'b']); + expect(truncated).toBe(false); + expect(mockGet).toHaveBeenCalledTimes(2); + }); + + it('returns an empty array for a note with no tags', async () => { + mockGet.mockResolvedValueOnce({ items: [], has_more: false }); + + const { titles, truncated } = await repo.getTagsForNote('note1'); + + expect(titles).toEqual([]); + expect(truncated).toBe(false); + }); + + it('reports truncated when the fetch fails partway through', async () => { + mockGet + .mockResolvedValueOnce({ items: [{ title: 'a' }], has_more: true }) + .mockRejectedValueOnce(new Error('network error')); + + const { titles, truncated } = await repo.getTagsForNote('note1'); + + expect(titles).toEqual(['a']); + expect(truncated).toBe(true); + }); + + it('reports truncated when it hits the page safety cap for a note whose tag list never reports has_more: false', async () => { + mockGet.mockResolvedValue({ items: [{ title: 'a' }], has_more: true }); + + const { titles, truncated } = await repo.getTagsForNote('note1'); + + expect(titles).toHaveLength(100); + expect(truncated).toBe(true); + expect(mockGet).toHaveBeenCalledTimes(100); + }); + }); }); diff --git a/src/data/TagRepository.ts b/src/data/TagRepository.ts index d8ae718..65cadd4 100644 --- a/src/data/TagRepository.ts +++ b/src/data/TagRepository.ts @@ -1,6 +1,8 @@ import joplin from 'api'; export class TagRepository { + private static readonly MAX_PAGES = 100; + /** * Builds a map of note IDs to their tag titles by fetching all tags and their associated notes. * @param maxTags - maximum tags to fetch before truncating (default 1000). @@ -58,4 +60,37 @@ export class TagRepository { return { map: noteTagsMap, truncated: tags.length >= maxTags ? true : false }; } + + public async getTagsForNote(noteId: string): Promise<{ titles: string[]; truncated: boolean }> { + const titles: string[] = []; + let page = 1; + let hasMore = true; + + while (hasMore && page <= TagRepository.MAX_PAGES) { + try { + const response = await joplin.data.get(['notes', noteId, 'tags'], { + fields: ['title'], + page, + limit: 100, + }); + for (const tag of response.items ?? []) { + titles.push(tag.title); + } + hasMore = response.has_more === true; + page++; + } catch (error) { + console.error(`Failed to fetch tags for note ${noteId}:`, error); + return { titles, truncated: true }; + } + } + + if (page > TagRepository.MAX_PAGES) { + console.info( + `Tag fetch for note ${noteId} hit the ${TagRepository.MAX_PAGES}-page safety cap; returning ${titles.length} tags.` + ); + return { titles, truncated: true }; + } + + return { titles, truncated: false }; + } } diff --git a/src/index.ts b/src/index.ts index e799012..fe56c15 100644 --- a/src/index.ts +++ b/src/index.ts @@ -4,13 +4,19 @@ import { initializeAiNoteGraphPanel, showAiNoteGraphPanel, postGraphData, + postGraphPatch, postStatus, postProgress, } from './ui/webview'; import { NoteRepository } from './data/NoteRepository'; import { NotePreprocessor } from './data/NotePreprocessor'; +import { EventsRepository } from './data/EventsRepository'; +import { GraphCacheRepository } from './data/Database/GraphCacheRepository'; +import { GraphBuilder } from './services/graph/GraphBuilder'; import { Note } from './data/Types'; import { AnalysisController } from './services/AnalysisController'; +import { IncrementalUpdater } from './services/sync/IncrementalUpdater'; +import { WorkspaceListener } from './services/sync/WorkspaceListener'; import { registerGraphSettings, isAiAnalysisEnabled, @@ -21,8 +27,8 @@ import { const SHOW_NOTE_GRAPH_COMMAND = 'showNoteGraph'; const SHOW_NOTE_GRAPH_MENU_ITEM = 'showNoteGraphMenuItem'; -const analysisController = new AnalysisController(); -let lastLoadedNotes: Note[] | null = null; +const graphCache = new GraphCacheRepository(); +const analysisController = new AnalysisController(new GraphBuilder(), graphCache); /** * Loads all notes from the Joplin API and enriches them with links and tags. @@ -57,19 +63,52 @@ const runSemanticAnalysis = async (notes: Note[]): Promise => { } }; +const performFullReload = async (): Promise => { + const enrichedNotes = await loadNotes(); + console.info(`Loaded ${enrichedNotes.length} notes.`); + await postGraphData(analysisController.buildStructural(enrichedNotes)); + await runSemanticAnalysis(enrichedNotes); +}; + +const incrementalUpdater = new IncrementalUpdater( + analysisController, + (diff, graphData) => { + postGraphPatch(diff, graphData).catch((e) => { + console.error('Failed to push graph patch to panel:', e); + }); + }, + performFullReload, + new NoteRepository(), + new NotePreprocessor(), + new EventsRepository(), + graphCache +); +const workspaceListener = new WorkspaceListener(incrementalUpdater); + const noteGraphCommand = { name: SHOW_NOTE_GRAPH_COMMAND, label: 'Show Note Graph', execute: async () => { try { - const enrichedNotes = await loadNotes(); - console.info(`Loaded ${enrichedNotes.length} notes.`); - lastLoadedNotes = enrichedNotes; + if (analysisController.hasNotes()) { + await showAiNoteGraphPanel(); + return; + } - await postGraphData(analysisController.buildStructural(enrichedNotes)); - await showAiNoteGraphPanel(); + const cached = await analysisController.loadFromCache(); + if (cached) { + console.info(`Loaded graph from cache: ${cached.nodes.length} notes, no recompute.`); + await postGraphData(cached); + await showAiNoteGraphPanel(); + await postStatus('Loaded from local cache - not recomputed. Refreshes as you edit or sync.'); + incrementalUpdater.handleSyncComplete().catch((e) => { + console.error('Post-cache-load sync sweep failed:', e); + }); + return; + } - await runSemanticAnalysis(enrichedNotes); + await showAiNoteGraphPanel(); + await performFullReload(); } catch (error) { console.error('Failed to load note graph:', error); } @@ -82,13 +121,16 @@ const noteGraphCommand = { * already-embedded vectors. No-ops if the graph hasn't been opened yet. */ const handleSettingsChange = async (event: { keys: string[] }): Promise => { - if (!lastLoadedNotes || !event.keys.some((key) => NOTE_GRAPH_SETTING_KEYS.includes(key))) { + if ( + !analysisController.hasNotes() || + !event.keys.some((key) => NOTE_GRAPH_SETTING_KEYS.includes(key)) + ) { return; } try { if (event.keys.includes(AI_ANALYSIS_ENABLED_KEY)) { - await runSemanticAnalysis(lastLoadedNotes); + await runSemanticAnalysis(analysisController.getCurrentNotes()); return; } @@ -128,5 +170,6 @@ joplin.plugins.register({ await initializeAiNoteGraphPanel(); await registerCommands(); await registerMenuItems(); + await workspaceListener.register(); }, }); diff --git a/src/services/AnalysisController.test.ts b/src/services/AnalysisController.test.ts index 2d5eee1..5de3f79 100644 --- a/src/services/AnalysisController.test.ts +++ b/src/services/AnalysisController.test.ts @@ -1,5 +1,6 @@ import { AnalysisController } from './AnalysisController'; import { GraphBuilder } from './graph/GraphBuilder'; +import { GraphCacheRepository } from '../data/Database/GraphCacheRepository'; import { ProviderResolver } from './embeddings/ProviderResolver'; import { EmbeddingOrchestrator } from './embeddings/Orchestrator'; import { isAiAnalysisEnabled, getSimilaritySettings } from './settings/GraphSettings'; @@ -13,8 +14,10 @@ jest.mock('./settings/GraphSettings'); jest.mock('../data/Database/VectorRepository', () => ({ VectorRepository: jest.fn(), })); +jest.mock('../data/Database/GraphCacheRepository'); const MockGraphBuilder = GraphBuilder as jest.MockedClass; +const MockGraphCacheRepository = GraphCacheRepository as jest.MockedClass; const MockProviderResolver = ProviderResolver as jest.Mocked; const MockOrchestrator = EmbeddingOrchestrator as jest.MockedClass; const mockIsAiAnalysisEnabled = isAiAnalysisEnabled as jest.Mock; @@ -55,6 +58,7 @@ function deferredEmbedResult(): { describe('AnalysisController', () => { let mockBuilder: jest.Mocked; + let mockGraphCache: jest.Mocked; let controller: AnalysisController; let mockOrchestratorInstance: { setProvider: jest.Mock; @@ -69,7 +73,10 @@ describe('AnalysisController', () => { mockBuilder.build.mockReturnValue({ nodes: [], edges: [] }); mockBuilder.buildWithSimilarity.mockResolvedValue({ nodes: [], edges: [] }); mockGetSimilaritySettings.mockResolvedValue({ threshold: 0.5, topK: 5 }); - controller = new AnalysisController(mockBuilder); + mockGraphCache = new MockGraphCacheRepository() as jest.Mocked; + mockGraphCache.saveGraph.mockResolvedValue(undefined); + mockGraphCache.loadGraph.mockResolvedValue(null); + controller = new AnalysisController(mockBuilder, mockGraphCache); mockOrchestratorInstance = { setProvider: jest.fn(), @@ -179,6 +186,33 @@ describe('AnalysisController', () => { expect(firstResult).toBeNull(); }); + it('skips the structural fallback build entirely for a run superseded before its embed attempt resolves', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(true); + + let rejectStaleProvider!: (e: Error) => void; + const staleProviderResolution = new Promise((_, reject) => { + rejectStaleProvider = reject; + }); + MockProviderResolver.resolveWithValidation + .mockReturnValueOnce(staleProviderResolution) + .mockResolvedValueOnce(fakeProvider); + mockOrchestratorInstance.embedNotes.mockResolvedValue({ + embeddedNotes: [{ note: note('b'), embedding: [0, 1] }], + errors: [], + }); + + const staleCall = controller.embedAndBuildSemantic([note('a')]); + const newerResult = await controller.embedAndBuildSemantic([note('b')]); + expect(newerResult?.usedAi).toBe(true); + + mockBuilder.build.mockClear(); + rejectStaleProvider(new Error('index not ready')); + const staleResult = await staleCall; + + expect(staleResult).toBeNull(); + expect(mockBuilder.build).not.toHaveBeenCalled(); + }); + it('wires an onProgress callback into the orchestrator when provided', async () => { mockIsAiAnalysisEnabled.mockResolvedValue(true); MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); @@ -258,5 +292,442 @@ describe('AnalysisController', () => { 3 ); }); + + it('cannot pair stale embedded vectors with a newer note list after a structural rebuild', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + mockOrchestratorInstance.embedNotes.mockResolvedValue({ + embeddedNotes: [{ note: note('a'), embedding: [1, 0] }], + errors: [], + }); + await controller.embedAndBuildSemantic([note('a')]); + + controller.buildStructural([note('a'), note('b')]); + jest.clearAllMocks(); + + const result = await controller.recompute(); + + expect(result).toBeNull(); + expect(mockBuilder.buildWithSimilarity).not.toHaveBeenCalled(); + }); + + it('cannot pair stale embedded vectors with a newer note list after a re-embed fails', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + mockOrchestratorInstance.embedNotes.mockResolvedValue({ + embeddedNotes: [{ note: note('a'), embedding: [1, 0] }], + errors: [], + }); + await controller.embedAndBuildSemantic([note('a')]); + + MockProviderResolver.resolveWithValidation.mockRejectedValue(new Error('index not ready')); + await controller.embedAndBuildSemantic([note('a'), note('b')]); + jest.clearAllMocks(); + + const result = await controller.recompute(); + + expect(result).toBeNull(); + expect(mockBuilder.buildWithSimilarity).not.toHaveBeenCalled(); + }); + + it('does not let a concurrent recompute() pair fresh notes with stale vectors while a re-embed is still in flight', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + const embeddedA = { note: note('a'), embedding: [1, 0] }; + mockOrchestratorInstance.embedNotes.mockResolvedValue({ embeddedNotes: [embeddedA], errors: [] }); + await controller.embedAndBuildSemantic([note('a')]); + jest.clearAllMocks(); + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + mockGetSimilaritySettings.mockResolvedValue({ threshold: 0.5, topK: 5 }); + + const deferred = deferredEmbedResult(); + mockOrchestratorInstance.embedNotes.mockReturnValueOnce(deferred.promise); + + const inFlight = controller.embedAndBuildSemantic([note('a'), note('b')]); + const recomputeResult = await controller.recompute(); + + expect(mockBuilder.buildWithSimilarity).toHaveBeenCalledWith([note('a')], [embeddedA], 0.5, 5); + expect(recomputeResult).not.toBeNull(); + + deferred.resolve({ + embeddedNotes: [embeddedA, { note: note('b'), embedding: [0, 1] }], + errors: [], + }); + expect(await inFlight).toBeNull(); + }); + }); + + describe('hasNotes / getCurrentNotes', () => { + it('has no notes and an empty list before anything is built or loaded', () => { + expect(controller.hasNotes()).toBe(false); + expect(controller.getCurrentNotes()).toEqual([]); + }); + + it('reflects the notes from the last buildStructural call', () => { + const notes = [note('a'), note('b')]; + controller.buildStructural(notes); + + expect(controller.hasNotes()).toBe(true); + expect(controller.getCurrentNotes()).toEqual(notes); + }); + }); + + describe('buildStructural cache persistence', () => { + it('persists the built graph to the cache', () => { + const notes = [note('a')]; + const graphData = { nodes: [], edges: [] }; + mockBuilder.build.mockReturnValue(graphData); + + controller.buildStructural(notes); + + expect(mockGraphCache.saveGraph).toHaveBeenCalledWith(notes, graphData); + }); + }); + + describe('loadFromCache', () => { + it('returns null and touches no state when nothing has been cached', async () => { + mockGraphCache.loadGraph.mockResolvedValue(null); + + const result = await controller.loadFromCache(); + + expect(result).toBeNull(); + expect(controller.hasNotes()).toBe(false); + }); + + it('seeds notes and returns the cached graph on a hit', async () => { + const notes = [note('a')]; + const graphData = { nodes: [], edges: [] }; + mockGraphCache.loadGraph.mockResolvedValue({ notes, graphData }); + + const result = await controller.loadFromCache(); + + expect(result).toBe(graphData); + expect(controller.hasNotes()).toBe(true); + expect(controller.getCurrentNotes()).toEqual(notes); + }); + + it('returns null instead of throwing when the cache read fails', async () => { + mockGraphCache.loadGraph.mockRejectedValue(new Error('disk error')); + + const result = await controller.loadFromCache(); + + expect(result).toBeNull(); + }); + }); + + describe('applyDelta', () => { + it('returns null when nothing has been loaded yet', async () => { + const result = await controller.applyDelta([note('a')], []); + expect(result).toBeNull(); + expect(mockBuilder.build).not.toHaveBeenCalled(); + }); + + it('adds a new note to the current list and rebuilds', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(false); + controller.buildStructural([note('a')]); + jest.clearAllMocks(); + mockIsAiAnalysisEnabled.mockResolvedValue(false); + const graphData = { nodes: [], edges: [] }; + mockBuilder.build.mockReturnValue(graphData); + + const result = await controller.applyDelta([note('b')], []); + + expect(result).toBe(graphData); + expect(mockBuilder.build).toHaveBeenCalledWith( + expect.arrayContaining([expect.objectContaining({ id: 'a' }), expect.objectContaining({ id: 'b' })]) + ); + expect(controller.getCurrentNotes()).toHaveLength(2); + }); + + it('removes a note from the current list and rebuilds', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(false); + controller.buildStructural([note('a'), note('b')]); + jest.clearAllMocks(); + mockIsAiAnalysisEnabled.mockResolvedValue(false); + + await controller.applyDelta([], ['a']); + + expect(controller.getCurrentNotes()).toEqual([note('b')]); + }); + + it('is a no-op when the upserted note is unchanged and nothing was removed', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(false); + controller.buildStructural([note('a')]); + jest.clearAllMocks(); + + const result = await controller.applyDelta([note('a')], []); + + expect(result).toBeNull(); + expect(mockBuilder.build).not.toHaveBeenCalled(); + }); + + it('rebuilds when an upserted note has a newer updated_time even with the same ID', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(false); + controller.buildStructural([note('a')]); + jest.clearAllMocks(); + mockIsAiAnalysisEnabled.mockResolvedValue(false); + const changedNote = { ...note('a'), updated_time: 999 }; + + const result = await controller.applyDelta([changedNote], []); + + expect(result).not.toBeNull(); + expect(controller.getCurrentNotes()[0].updated_time).toBe(999); + }); + + it('discards an in-flight embedAndBuildSemantic result that resolves after a delta lands, instead of clobbering the merge', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + controller.buildStructural([note('a')]); + jest.clearAllMocks(); + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + + const staleEmbed = deferredEmbedResult(); + mockOrchestratorInstance.embedNotes + .mockReturnValueOnce(staleEmbed.promise) + .mockResolvedValueOnce({ + embeddedNotes: [ + { note: note('a'), embedding: [1, 0] }, + { note: note('b'), embedding: [0, 1] }, + ], + errors: [], + }); + + const staleCall = controller.embedAndBuildSemantic([note('a')]); + const deltaResult = await controller.applyDelta([note('b')], []); + expect(deltaResult).not.toBeNull(); + expect(controller.getCurrentNotes()).toHaveLength(2); + + staleEmbed.resolve({ embeddedNotes: [{ note: note('a'), embedding: [1, 0] }], errors: [] }); + const staleResult = await staleCall; + + expect(staleResult).toBeNull(); + expect(controller.getCurrentNotes()).toHaveLength(2); + }); + + it('flags a delta as retryable when a newer run supersedes it before it resolves, instead of silently dropping the edit', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + controller.buildStructural([note('a')]); + jest.clearAllMocks(); + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + + const staleEmbed = deferredEmbedResult(); + mockOrchestratorInstance.embedNotes.mockReturnValueOnce(staleEmbed.promise); + + const deltaCall = controller.applyDelta([note('b')], []); + controller.buildStructural([note('a')]); + + staleEmbed.resolve({ + embeddedNotes: [ + { note: note('a'), embedding: [1, 0] }, + { note: note('b'), embedding: [0, 1] }, + ], + errors: [], + }); + const deltaResult = await deltaCall; + + expect(deltaResult).toBeNull(); + expect(controller.wasLastDeltaSkippedForRetry()).toBe(true); + }); + + it('detects a tag-only change even when updated_time is unchanged', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(false); + const original = { ...note('a'), tags: ['x'] }; + controller.buildStructural([original]); + jest.clearAllMocks(); + mockIsAiAnalysisEnabled.mockResolvedValue(false); + const retagged = { ...original, tags: ['y'] }; + + const result = await controller.applyDelta([retagged], []); + + expect(result).not.toBeNull(); + expect(controller.getCurrentNotes()[0].tags).toEqual(['y']); + }); + + it('is a no-op when the same tags arrive in a different order', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(false); + const original = { ...note('a'), tags: ['x', 'y'] }; + controller.buildStructural([original]); + jest.clearAllMocks(); + + const reordered = { ...original, tags: ['y', 'x'] }; + const result = await controller.applyDelta([reordered], []); + + expect(result).toBeNull(); + expect(mockBuilder.build).not.toHaveBeenCalled(); + }); + + it('does not downgrade an existing semantic graph to structural when a re-embed fails transiently', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + mockOrchestratorInstance.embedNotes.mockResolvedValue({ + embeddedNotes: [{ note: note('a'), embedding: [1, 0] }], + errors: [], + }); + mockBuilder.buildWithSimilarity.mockResolvedValue({ + nodes: [], + edges: [{ data: { id: 'a::b::semantic', source: 'a', target: 'b', type: 'semantic' } }], + }); + await controller.embedAndBuildSemantic([note('a')]); + jest.clearAllMocks(); + + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockRejectedValue(new Error('index not ready')); + + const result = await controller.applyDelta([note('b')], []); + + expect(result).toBeNull(); + expect(mockBuilder.build).not.toHaveBeenCalled(); + expect(controller.getCurrentNotes()).toHaveLength(1); + expect(controller.wasLastDeltaSkippedForRetry()).toBe(true); + }); + + it('does not flag a delta as retryable when it was a genuine no-op or plain removal', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(false); + controller.buildStructural([note('a')]); + jest.clearAllMocks(); + mockIsAiAnalysisEnabled.mockResolvedValue(false); + + await controller.applyDelta([note('a')], []); + expect(controller.wasLastDeltaSkippedForRetry()).toBe(false); + + await controller.applyDelta([], ['a']); + expect(controller.wasLastDeltaSkippedForRetry()).toBe(false); + }); + + it('clears the retryable flag once a later delta commits successfully', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + mockOrchestratorInstance.embedNotes.mockResolvedValue({ + embeddedNotes: [{ note: note('a'), embedding: [1, 0] }], + errors: [], + }); + mockBuilder.buildWithSimilarity.mockResolvedValue({ + nodes: [], + edges: [{ data: { id: 'a::b::semantic', source: 'a', target: 'b', type: 'semantic' } }], + }); + await controller.embedAndBuildSemantic([note('a')]); + jest.clearAllMocks(); + + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockRejectedValueOnce(new Error('index not ready')); + await controller.applyDelta([note('b')], []); + expect(controller.wasLastDeltaSkippedForRetry()).toBe(true); + + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + mockOrchestratorInstance.embedNotes.mockResolvedValue({ + embeddedNotes: [ + { note: note('a'), embedding: [1, 0] }, + { note: note('b'), embedding: [0, 1] }, + ], + errors: [], + }); + await controller.applyDelta([note('b')], []); + + expect(controller.wasLastDeltaSkippedForRetry()).toBe(false); + }); + + it('retries cleanly on the next delta after a skipped downgrade, once AI recovers', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + mockOrchestratorInstance.embedNotes.mockResolvedValue({ + embeddedNotes: [{ note: note('a'), embedding: [1, 0] }], + errors: [], + }); + mockBuilder.buildWithSimilarity.mockResolvedValue({ + nodes: [], + edges: [{ data: { id: 'a::b::semantic', source: 'a', target: 'b', type: 'semantic' } }], + }); + await controller.embedAndBuildSemantic([note('a')]); + jest.clearAllMocks(); + + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockRejectedValueOnce(new Error('index not ready')); + const skipped = await controller.applyDelta([note('b')], []); + expect(skipped).toBeNull(); + expect(controller.getCurrentNotes()).toHaveLength(1); + + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + mockOrchestratorInstance.embedNotes.mockResolvedValue({ + embeddedNotes: [ + { note: note('a'), embedding: [1, 0] }, + { note: note('b'), embedding: [0, 1] }, + ], + errors: [], + }); + + const retried = await controller.applyDelta([note('b')], []); + + expect(retried).not.toBeNull(); + expect(controller.getCurrentNotes()).toHaveLength(2); + }); + + it('downgrades to structural when AI is simply off, instead of mistaking that for a failed re-embed', async () => { + mockIsAiAnalysisEnabled.mockResolvedValue(true); + MockProviderResolver.resolveWithValidation.mockResolvedValue(fakeProvider); + mockOrchestratorInstance.embedNotes.mockResolvedValue({ + embeddedNotes: [{ note: note('a'), embedding: [1, 0] }], + errors: [], + }); + mockBuilder.buildWithSimilarity.mockResolvedValue({ + nodes: [], + edges: [{ data: { id: 'a::b::semantic', source: 'a', target: 'b', type: 'semantic' } }], + }); + await controller.embedAndBuildSemantic([note('a')]); + jest.clearAllMocks(); + + mockIsAiAnalysisEnabled.mockResolvedValue(false); + mockBuilder.build.mockReturnValue({ nodes: [], edges: [] }); + + const result = await controller.applyDelta([note('b')], []); + + expect(result).not.toBeNull(); + expect(controller.getCurrentNotes()).toHaveLength(2); + expect(controller.wasLastDeltaSkippedForRetry()).toBe(false); + expect(MockProviderResolver.resolveWithValidation).not.toHaveBeenCalled(); + }); + }); + + describe('getLastDiff', () => { + it('is null before anything has been built', () => { + expect(controller.getLastDiff()).toBeNull(); + }); + + it('treats the first build as entirely new (no previous graph to diff against)', () => { + const graphData = { + nodes: [{ data: { id: 'a', label: 'a', noteId: 'a', degree: 0, community: 0, size: 1 } }], + edges: [], + }; + mockBuilder.build.mockReturnValue(graphData); + + controller.buildStructural([note('a')]); + + expect(controller.getLastDiff()).toEqual({ + upsertedNodes: graphData.nodes, + upsertedEdges: [], + removedNodeIds: [], + removedEdgeIds: [], + }); + }); + + it('reports only what changed between two builds', () => { + const nodeA = { data: { id: 'a', label: 'a', noteId: 'a', degree: 0, community: 0, size: 1 } }; + const nodeB = { data: { id: 'b', label: 'b', noteId: 'b', degree: 0, community: 0, size: 1 } }; + mockBuilder.build.mockReturnValueOnce({ nodes: [nodeA], edges: [] }); + controller.buildStructural([note('a')]); + + mockBuilder.build.mockReturnValueOnce({ nodes: [nodeA, nodeB], edges: [] }); + controller.buildStructural([note('a'), note('b')]); + + expect(controller.getLastDiff()).toEqual({ + upsertedNodes: [nodeB], + upsertedEdges: [], + removedNodeIds: [], + removedEdgeIds: [], + }); + }); }); }); diff --git a/src/services/AnalysisController.ts b/src/services/AnalysisController.ts index e8ecae1..d4b1999 100644 --- a/src/services/AnalysisController.ts +++ b/src/services/AnalysisController.ts @@ -1,7 +1,9 @@ import { Note } from '../data/Types'; import { GraphBuilder } from './graph/GraphBuilder'; import { GraphData } from './graph/types'; +import { GraphDiffer, GraphDiff } from './graph/GraphDiffer'; import { VectorRepository } from '../data/Database/VectorRepository'; +import { GraphCacheRepository } from '../data/Database/GraphCacheRepository'; import { ProviderResolver } from './embeddings/ProviderResolver'; import { EmbeddingOrchestrator } from './embeddings/Orchestrator'; import { EmbeddedNote, EmbeddingProvider, BatchProgress } from './embeddings/Types'; @@ -22,12 +24,53 @@ export interface SemanticBuildResult { export class AnalysisController { private lastNotes: Note[] | null = null; private lastEmbeddedNotes: EmbeddedNote[] | null = null; + private lastGraphData: GraphData | null = null; + private lastDiff: GraphDiff | null = null; private runToken = 0; + private lastDeltaSkippedForRetry = false; - public constructor(private readonly builder = new GraphBuilder()) {} + public constructor( + private readonly builder = new GraphBuilder(), + private readonly graphCache: GraphCacheRepository = new GraphCacheRepository(), + private readonly graphDiffer: GraphDiffer = new GraphDiffer() + ) {} + + public getLastDiff(): GraphDiff | null { + return this.lastDiff; + } + + public wasLastDeltaSkippedForRetry(): boolean { + return this.lastDeltaSkippedForRetry; + } + + public hasNotes(): boolean { + return this.lastNotes !== null; + } + + public getCurrentNotes(): Note[] { + return this.lastNotes ?? []; + } + + public async loadFromCache(): Promise { + try { + const cached = await this.graphCache.loadGraph(); + if (!cached) return null; + this.lastNotes = cached.notes; + this.lastGraphData = cached.graphData; + return cached.graphData; + } catch (e) { + console.error('Failed to load cached graph, starting fresh:', e); + return null; + } + } public buildStructural(notes: Note[]): GraphData { - return this.builder.build(notes); + ++this.runToken; + this.lastNotes = notes; + this.lastEmbeddedNotes = null; + const graphData = this.builder.build(notes); + this.commitGraphData(graphData); + return graphData; } /** @@ -47,19 +90,41 @@ export class AnalysisController { ): Promise { const token = ++this.runToken; const guardedProgress = onProgress ? this.guardStaleProgress(token, onProgress) : undefined; - const { embeddedNotes, reason } = await this.tryEmbed(notes, guardedProgress); + return this.buildFrom(notes, token, { onProgress: guardedProgress, commitNotes: true }); + } + private async buildFrom( + notes: Note[], + token: number, + options: { + onProgress?: (progress: BatchProgress) => void; + avoidSemanticDowngrade?: boolean; + commitNotes?: boolean; + } + ): Promise { + const hadSemanticGraph = this.hasSemanticEdges(); + const { embeddedNotes, reason, aiWasEnabled } = await this.tryEmbed(notes, options.onProgress); if (token !== this.runToken) { + if (options.avoidSemanticDowngrade) this.lastDeltaSkippedForRetry = true; return null; } if (!embeddedNotes) { - return { graphData: this.builder.build(notes), usedAi: false, fallbackReason: reason }; + if (options.avoidSemanticDowngrade && hadSemanticGraph && aiWasEnabled) { + console.info( + 'Incremental update: AI re-embed failed; keeping the existing semantic graph instead of downgrading it.', + reason + ); + this.lastDeltaSkippedForRetry = true; + return null; + } + const graphData = this.builder.build(notes); + if (options.commitNotes) this.lastNotes = notes; + this.lastEmbeddedNotes = null; + this.commitGraphData(graphData); + return { graphData, usedAi: false, fallbackReason: reason }; } - this.lastNotes = notes; - this.lastEmbeddedNotes = embeddedNotes; - console.info( `AI analysis: ${embeddedNotes.length}/${notes.length} notes embedded, building semantic graph.` ); @@ -70,6 +135,14 @@ export class AnalysisController { threshold, topK ); + + if (token !== this.runToken) { + if (options.avoidSemanticDowngrade) this.lastDeltaSkippedForRetry = true; + return null; + } + if (options.commitNotes) this.lastNotes = notes; + this.lastEmbeddedNotes = embeddedNotes; + this.commitGraphData(graphData); return { graphData, usedAi: true }; } @@ -78,16 +151,91 @@ export class AnalysisController { if (!this.lastNotes || !this.lastEmbeddedNotes) { return null; } + const token = ++this.runToken; const { threshold, topK } = await getSimilaritySettings(); console.info( `Recomputing graph: threshold=${threshold}, topK=${topK}, ${this.lastEmbeddedNotes.length} cached vectors.` ); - return this.builder.buildWithSimilarity( + const graphData = await this.builder.buildWithSimilarity( this.lastNotes, this.lastEmbeddedNotes, threshold, topK ); + + if (token !== this.runToken) return null; + this.commitGraphData(graphData); + return graphData; + } + + public async applyDelta(upserts: Note[], removedIds: string[]): Promise { + this.lastDeltaSkippedForRetry = false; + if (!this.lastNotes) return null; + + const { merged, changed } = this.mergeNotes(this.lastNotes, upserts, removedIds); + if (!changed) return null; + + const token = ++this.runToken; + const result = await this.buildFrom(merged, token, { + avoidSemanticDowngrade: true, + commitNotes: true, + }); + return result ? result.graphData : null; + } + + private hasSemanticEdges(): boolean { + return !!this.lastGraphData?.edges.some((e) => e.data.type === 'semantic'); + } + + private commitGraphData(graphData: GraphData): void { + this.lastDiff = this.graphDiffer.computeDiff(this.lastGraphData, graphData); + this.lastGraphData = graphData; + this.persistCache(); + } + + private persistCache(): void { + if (!this.lastNotes || !this.lastGraphData) return; + this.graphCache.saveGraph(this.lastNotes, this.lastGraphData).catch((e) => { + console.error('Failed to persist graph cache:', e); + }); + } + + private mergeNotes( + current: Note[], + upserts: Note[], + removedIds: string[] + ): { merged: Note[]; changed: boolean } { + const byId = new Map(current.map((n) => [n.id, n])); + let changed = false; + + for (const id of removedIds) { + if (byId.delete(id)) changed = true; + } + for (const note of upserts) { + const existing = byId.get(note.id); + if (!existing || !this.notesEqual(existing, note)) { + changed = true; + } + byId.set(note.id, note); + } + + return { merged: changed ? Array.from(byId.values()) : current, changed }; + } + + private notesEqual(a: Note, b: Note): boolean { + return ( + a.updated_time === b.updated_time && + this.sameStringSet(a.tags, b.tags) && + this.sameStringSet(a.links, b.links) + ); + } + + private sameStringSet(a: string[] | undefined, b: string[] | undefined): boolean { + const aValues = a ?? []; + const bValues = b ?? []; + if (aValues.length !== bValues.length) return false; + const bSet = new Set(bValues); + return aValues.every((value) => bSet.has(value)); } /** Wraps a progress callback so it stops firing once a newer run supersedes `token` — otherwise a slow, superseded run could re-show the progress bar after a newer run already hid it by posting its finished graph. */ @@ -106,9 +254,9 @@ export class AnalysisController { private async tryEmbed( notes: Note[], onProgress?: (progress: BatchProgress) => void - ): Promise<{ embeddedNotes: EmbeddedNote[] | null; reason?: string }> { + ): Promise<{ embeddedNotes: EmbeddedNote[] | null; reason?: string; aiWasEnabled: boolean }> { if (!(await isAiAnalysisEnabled())) { - return { embeddedNotes: null }; + return { embeddedNotes: null, aiWasEnabled: false }; } let provider: EmbeddingProvider; @@ -117,7 +265,7 @@ export class AnalysisController { } catch (e) { const reason = e instanceof Error ? e.message : String(e); console.error('AI analysis unavailable, falling back to structural graph:', e); - return { embeddedNotes: null, reason }; + return { embeddedNotes: null, reason, aiWasEnabled: true }; } const orchestrator = new EmbeddingOrchestrator(); @@ -133,9 +281,9 @@ export class AnalysisController { 'AI analysis produced no embeddings, falling back to structural graph:', errors ); - return { embeddedNotes: null, reason: errors[0]?.error }; + return { embeddedNotes: null, reason: errors[0]?.error, aiWasEnabled: true }; } - return { embeddedNotes }; + return { embeddedNotes, aiWasEnabled: true }; } } diff --git a/src/services/graph/GraphBuilder.test.ts b/src/services/graph/GraphBuilder.test.ts index d6fcc4e..88965d3 100644 --- a/src/services/graph/GraphBuilder.test.ts +++ b/src/services/graph/GraphBuilder.test.ts @@ -59,7 +59,7 @@ describe('GraphBuilder', () => { expect(result.nodes[0].data.degree).toBe(1); expect(result.nodes[1].data.degree).toBe(1); expect(result.edges).toHaveLength(1); - expect(result.edges[0].data).toEqual({ source: 'a', target: 'b', type: 'link' }); + expect(result.edges[0].data).toEqual({ id: 'a::b::link', source: 'a', target: 'b', type: 'link' }); }); it('truncates long note labels to 64 chars', () => { @@ -84,7 +84,7 @@ describe('GraphBuilder', () => { const notes = [note('a', 'A'), note('b', 'B')]; const result = builder.build(notes); expect(result.edges).toHaveLength(1); - expect(result.edges[0].data).toEqual({ source: 'a', target: 'b', type: 'link' }); + expect(result.edges[0].data).toEqual({ id: 'a::b::link', source: 'a', target: 'b', type: 'link' }); }); it('applies the detected community and centrality size to each node', () => { @@ -148,10 +148,10 @@ describe('GraphBuilder', () => { { source: 'a', target: 'b', score: 0.8 }, ]); expect(result.edges).toContainEqual({ - data: { source: 'a', target: 'b', type: 'semantic' }, + data: { id: 'a::b::semantic', source: 'a', target: 'b', type: 'semantic' }, }); expect(result.edges).toContainEqual({ - data: { source: 'a', target: 'c', type: 'link' }, + data: { id: 'a::c::link', source: 'a', target: 'c', type: 'link' }, }); expect(result.edges).toHaveLength(2); }); diff --git a/src/services/graph/GraphBuilder.ts b/src/services/graph/GraphBuilder.ts index 16ee717..a7b79b0 100644 --- a/src/services/graph/GraphBuilder.ts +++ b/src/services/graph/GraphBuilder.ts @@ -2,7 +2,7 @@ import { Note } from '../../data/Types'; import { EdgeFactory } from '../similarity/EdgeFactory'; import { SimilarityEngine } from '../similarity/SimilarityEngine'; import { EmbeddedNote } from '../embeddings/Types'; -import { GraphData, GraphEdge, GraphNode } from './types'; +import { GraphData, GraphEdge, GraphNode, RenderedEdge } from './types'; import { LouvainDetector } from './LouvainDetector'; import { CentralityScorer } from './CentralityScorer'; @@ -62,7 +62,11 @@ export class GraphBuilder { this.logGraphStats(nodes, visibleEdges, degreeMap, communities); - return { nodes, edges: visibleEdges.map((e) => ({ data: e })) }; + return { nodes, edges: visibleEdges.map((e) => ({ data: this.toRenderedEdge(e) })) }; + } + + private toRenderedEdge(edge: GraphEdge): RenderedEdge { + return { ...edge, id: `${edge.source}::${edge.target}::${edge.type}` }; } /** Counts each note's connections, including notes an edge references that aren't in `notes`. */ diff --git a/src/services/graph/GraphDiffer.test.ts b/src/services/graph/GraphDiffer.test.ts new file mode 100644 index 0000000..2f465b4 --- /dev/null +++ b/src/services/graph/GraphDiffer.test.ts @@ -0,0 +1,121 @@ +import { GraphDiffer } from './GraphDiffer'; +import { GraphData } from './types'; + +function node(id: string, overrides: Partial = {}) { + return { + data: { + id, + label: id, + noteId: id, + degree: 0, + community: 0, + size: 1, + ...overrides, + }, + }; +} + +function edge(source: string, target: string, type: 'link' | 'tag' | 'semantic' = 'link') { + return { data: { id: `${source}::${target}::${type}`, source, target, type } }; +} + +describe('GraphDiffer', () => { + let differ: GraphDiffer; + + beforeEach(() => { + differ = new GraphDiffer(); + }); + + it('treats everything as upserted when there is no previous graph', () => { + const current: GraphData = { nodes: [node('a')], edges: [edge('a', 'b')] }; + + const diff = differ.computeDiff(null, current); + + expect(diff).toEqual({ + upsertedNodes: current.nodes, + upsertedEdges: current.edges, + removedNodeIds: [], + removedEdgeIds: [], + }); + }); + + it('reports no changes when the graph is identical', () => { + const graph: GraphData = { nodes: [node('a')], edges: [edge('a', 'b')] }; + + const diff = differ.computeDiff(graph, graph); + + expect(diff).toEqual({ + upsertedNodes: [], + upsertedEdges: [], + removedNodeIds: [], + removedEdgeIds: [], + }); + }); + + it('reports a brand-new node and edge as upserted', () => { + const previous: GraphData = { nodes: [node('a')], edges: [] }; + const current: GraphData = { nodes: [node('a'), node('b')], edges: [edge('a', 'b')] }; + + const diff = differ.computeDiff(previous, current); + + expect(diff.upsertedNodes).toEqual([node('b')]); + expect(diff.upsertedEdges).toEqual([edge('a', 'b')]); + expect(diff.removedNodeIds).toEqual([]); + expect(diff.removedEdgeIds).toEqual([]); + }); + + it('reports a node whose data changed (e.g. degree) as upserted even though its id is unchanged', () => { + const previous: GraphData = { nodes: [node('a', { degree: 1 })], edges: [] }; + const current: GraphData = { nodes: [node('a', { degree: 2 })], edges: [] }; + + const diff = differ.computeDiff(previous, current); + + expect(diff.upsertedNodes).toEqual([node('a', { degree: 2 })]); + }); + + it('does not report an unchanged node as upserted just because another node changed', () => { + const previous: GraphData = { + nodes: [node('a', { degree: 1 }), node('b', { degree: 1 })], + edges: [], + }; + const current: GraphData = { + nodes: [node('a', { degree: 2 }), node('b', { degree: 1 })], + edges: [], + }; + + const diff = differ.computeDiff(previous, current); + + expect(diff.upsertedNodes).toEqual([node('a', { degree: 2 })]); + }); + + it('reports a removed node and its dangling edge', () => { + const previous: GraphData = { + nodes: [node('a'), node('b')], + edges: [edge('a', 'b')], + }; + const current: GraphData = { nodes: [node('a')], edges: [] }; + + const diff = differ.computeDiff(previous, current); + + expect(diff.removedNodeIds).toEqual(['b']); + expect(diff.removedEdgeIds).toEqual(['a::b::link']); + expect(diff.upsertedNodes).toEqual([]); + expect(diff.upsertedEdges).toEqual([]); + }); + + it('distinguishes edges of different types between the same two notes', () => { + const previous: GraphData = { + nodes: [node('a'), node('b')], + edges: [edge('a', 'b', 'link')], + }; + const current: GraphData = { + nodes: [node('a'), node('b')], + edges: [edge('a', 'b', 'link'), edge('a', 'b', 'tag')], + }; + + const diff = differ.computeDiff(previous, current); + + expect(diff.upsertedEdges).toEqual([edge('a', 'b', 'tag')]); + expect(diff.removedEdgeIds).toEqual([]); + }); +}); diff --git a/src/services/graph/GraphDiffer.ts b/src/services/graph/GraphDiffer.ts new file mode 100644 index 0000000..8a3dcc7 --- /dev/null +++ b/src/services/graph/GraphDiffer.ts @@ -0,0 +1,51 @@ +import { GraphData, GraphNode, RenderedEdge } from './types'; + +export interface GraphDiff { + upsertedNodes: Array<{ data: GraphNode }>; + upsertedEdges: Array<{ data: RenderedEdge }>; + removedNodeIds: string[]; + removedEdgeIds: string[]; +} + +function dataEqual(a: T | undefined, b: T): boolean { + if (!a) return false; + const aRecord = a as unknown as Record; + const bRecord = b as unknown as Record; + const aKeys = Object.keys(aRecord); + const bKeys = Object.keys(bRecord); + if (aKeys.length !== bKeys.length) return false; + return aKeys.every((key) => aRecord[key] === bRecord[key]); +} + +export class GraphDiffer { + public computeDiff(previous: GraphData | null, current: GraphData): GraphDiff { + if (!previous) { + return { + upsertedNodes: current.nodes, + upsertedEdges: current.edges, + removedNodeIds: [], + removedEdgeIds: [], + }; + } + + const previousNodesById = new Map(previous.nodes.map((n) => [n.data.id, n.data])); + const previousEdgesById = new Map(previous.edges.map((e) => [e.data.id, e.data])); + const currentNodeIds = new Set(current.nodes.map((n) => n.data.id)); + const currentEdgeIds = new Set(current.edges.map((e) => e.data.id)); + + const upsertedNodes = current.nodes.filter( + (n) => !dataEqual(previousNodesById.get(n.data.id), n.data) + ); + const upsertedEdges = current.edges.filter( + (e) => !dataEqual(previousEdgesById.get(e.data.id), e.data) + ); + const removedNodeIds = Array.from(previousNodesById.keys()).filter( + (id) => !currentNodeIds.has(id) + ); + const removedEdgeIds = Array.from(previousEdgesById.keys()).filter( + (id) => !currentEdgeIds.has(id) + ); + + return { upsertedNodes, upsertedEdges, removedNodeIds, removedEdgeIds }; + } +} diff --git a/src/services/graph/types.ts b/src/services/graph/types.ts index 54e9fec..5107157 100644 --- a/src/services/graph/types.ts +++ b/src/services/graph/types.ts @@ -18,7 +18,11 @@ export interface GraphEdge { tagName?: string; } +export interface RenderedEdge extends GraphEdge { + id: string; +} + export interface GraphData { nodes: Array<{ data: GraphNode }>; - edges: Array<{ data: GraphEdge }>; + edges: Array<{ data: RenderedEdge }>; } diff --git a/src/services/similarity/EdgeFactory.test.ts b/src/services/similarity/EdgeFactory.test.ts index 61ef6ee..42f83af 100644 --- a/src/services/similarity/EdgeFactory.test.ts +++ b/src/services/similarity/EdgeFactory.test.ts @@ -64,6 +64,38 @@ describe('EdgeFactory', () => { expect(edges[0].tagName).toBe('t1, t2'); }); + it('keeps the same source/target for a tag edge regardless of note iteration order', () => { + const forward = factory.createEdges([ + note('a', 'A', [], ['shared']), + note('b', 'B', [], ['shared']), + ]); + const reversed = factory.createEdges([ + note('b', 'B', [], ['shared']), + note('a', 'A', [], ['shared']), + ]); + + expect(forward).toEqual([{ source: 'a', target: 'b', type: 'tag', tagName: 'shared' }]); + expect(reversed).toEqual([{ source: 'a', target: 'b', type: 'tag', tagName: 'shared' }]); + }); + + it('keeps the same tagName text regardless of note iteration order, for a pair sharing multiple tags', () => { + const forward = factory.createEdges([ + note('x', 'X', [], ['t2']), + note('a', 'A', [], ['t1', 't2']), + note('b', 'B', [], ['t1', 't2']), + ]); + const reversed = factory.createEdges([ + note('a', 'A', [], ['t1', 't2']), + note('b', 'B', [], ['t1', 't2']), + note('x', 'X', [], ['t2']), + ]); + + const forwardEdge = forward.find((e) => e.type === 'tag' && e.source === 'a' && e.target === 'b'); + const reversedEdge = reversed.find((e) => e.type === 'tag' && e.source === 'a' && e.target === 'b'); + + expect(forwardEdge?.tagName).toBe(reversedEdge?.tagName); + }); + it('creates separate tag edges for different pairs', () => { const edges = factory.createEdges([ note('a', 'A', [], ['t1']), diff --git a/src/services/similarity/EdgeFactory.ts b/src/services/similarity/EdgeFactory.ts index 0197272..0b77ef5 100644 --- a/src/services/similarity/EdgeFactory.ts +++ b/src/services/similarity/EdgeFactory.ts @@ -40,7 +40,7 @@ export class EdgeFactory { */ private createTagEdges(notes: Note[]): GraphEdge[] { const tagToNotes = this.groupNoteIdsByTag(notes); - const tagEdgeMap = new Map(); + const tagEdgeMap = new Map(); for (const [tagName, noteIds] of tagToNotes) { if (noteIds.length > 20) continue; @@ -49,24 +49,25 @@ export class EdgeFactory { for (let j = i + 1; j < noteIds.length; j++) { const a = noteIds[i]; const b = noteIds[j]; - const pairKey = a < b ? `${a}::${b}` : `${b}::${a}`; + const [source, target] = a < b ? [a, b] : [b, a]; + const pairKey = `${source}::${target}`; const existing = tagEdgeMap.get(pairKey); if (existing) { - existing.tagName = existing.tagName + ', ' + tagName; + existing.tagNames.push(tagName); } else { - tagEdgeMap.set(pairKey, { - source: a, - target: b, - type: 'tag', - tagName: tagName, - }); + tagEdgeMap.set(pairKey, { source, target, tagNames: [tagName] }); } } } } - return Array.from(tagEdgeMap.values()); + return Array.from(tagEdgeMap.values()).map((edge) => ({ + source: edge.source, + target: edge.target, + type: 'tag', + tagName: edge.tagNames.slice().sort().join(', '), + })); } private groupNoteIdsByTag(notes: Note[]): Map { diff --git a/src/services/sync/IncrementalUpdater.test.ts b/src/services/sync/IncrementalUpdater.test.ts new file mode 100644 index 0000000..77ffc4c --- /dev/null +++ b/src/services/sync/IncrementalUpdater.test.ts @@ -0,0 +1,576 @@ +import joplin from 'api'; +import { IncrementalUpdater } from './IncrementalUpdater'; +import { AnalysisController } from '../AnalysisController'; +import { NoteRepository } from '../../data/NoteRepository'; +import { NotePreprocessor } from '../../data/NotePreprocessor'; +import { EventsRepository } from '../../data/EventsRepository'; +import { GraphCacheRepository } from '../../data/Database/GraphCacheRepository'; +import { Note } from '../../data/Types'; + +jest.mock('../AnalysisController'); +jest.mock('../../data/NoteRepository'); +jest.mock('../../data/NotePreprocessor'); +jest.mock('../../data/EventsRepository'); +jest.mock('../../data/Database/GraphCacheRepository'); + +const MockAnalysisController = AnalysisController as jest.MockedClass; +const MockNoteRepository = NoteRepository as jest.MockedClass; +const MockPreprocessor = NotePreprocessor as jest.MockedClass; +const MockEventsRepository = EventsRepository as jest.MockedClass; +const MockGraphCacheRepository = GraphCacheRepository as jest.MockedClass; + +const COALESCE_WINDOW_MS = 1000; + +async function flushMicrotasks(n = 10): Promise { + for (let i = 0; i < n; i++) { + await Promise.resolve(); + } +} + +function note(id: string, updatedTime = 1): Note { + return { + id, + parent_id: 'p1', + title: id, + body: '', + created_time: 0, + updated_time: updatedTime, + }; +} + +describe('IncrementalUpdater', () => { + let analysisController: jest.Mocked; + let noteRepository: jest.Mocked; + let preprocessor: jest.Mocked; + let eventsRepository: jest.Mocked; + let graphCache: jest.Mocked; + let onGraphPatch: jest.Mock; + let onFullReloadNeeded: jest.Mock; + let checkAiEnabled: jest.Mock, []>; + let ai: { getIndexStatus: jest.Mock; getEmbeddings: jest.Mock }; + let updater: IncrementalUpdater; + + const fakeDiff = { + upsertedNodes: [], + upsertedEdges: [], + removedNodeIds: [], + removedEdgeIds: [], + }; + + beforeEach(() => { + jest.useFakeTimers(); + jest.clearAllMocks(); + + analysisController = new MockAnalysisController() as jest.Mocked; + analysisController.hasNotes.mockReturnValue(true); + analysisController.applyDelta.mockResolvedValue({ nodes: [], edges: [] }); + analysisController.getLastDiff.mockReturnValue(fakeDiff); + + noteRepository = new MockNoteRepository() as jest.Mocked; + preprocessor = new MockPreprocessor() as jest.Mocked; + preprocessor.processOne.mockImplementation(async (n) => n); + + eventsRepository = new MockEventsRepository() as jest.Mocked; + eventsRepository.getNoteEventsSince.mockResolvedValue({ events: [], cursor: undefined }); + + graphCache = new MockGraphCacheRepository() as jest.Mocked; + graphCache.loadEventsCursor.mockResolvedValue(null); + graphCache.saveEventsCursor.mockResolvedValue(undefined); + graphCache.loadEmbeddingsCursor.mockResolvedValue(null); + graphCache.saveEmbeddingsCursor.mockResolvedValue(undefined); + + onGraphPatch = jest.fn(); + onFullReloadNeeded = jest.fn().mockResolvedValue(undefined); + checkAiEnabled = jest.fn().mockResolvedValue(false); + + ai = joplin.ai as unknown as { getIndexStatus: jest.Mock; getEmbeddings: jest.Mock }; + ai.getIndexStatus.mockResolvedValue({ ready: true, state: 'ready', modelId: 'test-model' }); + ai.getEmbeddings.mockResolvedValue({ + modelId: 'test-model', + dimension: 2, + chunks: [], + nextCursor: undefined, + }); + + updater = new IncrementalUpdater( + analysisController, + onGraphPatch, + onFullReloadNeeded, + noteRepository, + preprocessor, + eventsRepository, + graphCache, + COALESCE_WINDOW_MS, + checkAiEnabled + ); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + describe('handleNoteChange', () => { + it('fetches, enriches, and applies an upsert after the coalescing window for a create/update event', async () => { + noteRepository.getNote.mockResolvedValue(note('a')); + + updater.handleNoteChange({ id: 'a', event: 1 }); + expect(noteRepository.getNote).not.toHaveBeenCalled(); + + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + expect(noteRepository.getNote).toHaveBeenCalledWith('a'); + expect(preprocessor.processOne).toHaveBeenCalledWith(note('a')); + expect(analysisController.applyDelta).toHaveBeenCalledWith([note('a')], []); + expect(onGraphPatch).toHaveBeenCalledWith(fakeDiff, { nodes: [], edges: [] }); + }); + + it('logs a summary once a new note is successfully upserted', async () => { + const consoleInfoSpy = jest.spyOn(console, 'info').mockImplementation(() => undefined); + noteRepository.getNote.mockResolvedValue(note('a')); + + updater.handleNoteChange({ id: 'a', event: 1 }); + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + expect(consoleInfoSpy).toHaveBeenCalledWith('Incremental update applied: 1 upserted, 0 removed.'); + consoleInfoSpy.mockRestore(); + }); + + it('applies a removal for a delete event without fetching the note', async () => { + updater.handleNoteChange({ id: 'a', event: 3 }); + + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + expect(noteRepository.getNote).not.toHaveBeenCalled(); + expect(analysisController.applyDelta).toHaveBeenCalledWith([], ['a']); + }); + + it('coalesces multiple events for the same note into a single fetch', async () => { + noteRepository.getNote.mockResolvedValue(note('a')); + + updater.handleNoteChange({ id: 'a', event: 2 }); + updater.handleNoteChange({ id: 'a', event: 2 }); + updater.handleNoteChange({ id: 'a', event: 2 }); + + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + expect(noteRepository.getNote).toHaveBeenCalledTimes(1); + expect(analysisController.applyDelta).toHaveBeenCalledTimes(1); + }); + + it('nets a create-then-delete for the same note within the window to a removal only', async () => { + updater.handleNoteChange({ id: 'a', event: 1 }); + updater.handleNoteChange({ id: 'a', event: 3 }); + + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + expect(noteRepository.getNote).not.toHaveBeenCalled(); + expect(analysisController.applyDelta).toHaveBeenCalledWith([], ['a']); + }); + + it('does nothing if no note list has been loaded yet by the time the window elapses', async () => { + analysisController.hasNotes.mockReturnValue(false); + + updater.handleNoteChange({ id: 'a', event: 2 }); + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + expect(noteRepository.getNote).not.toHaveBeenCalled(); + expect(analysisController.applyDelta).not.toHaveBeenCalled(); + }); + + it('does not push an update or log a summary when applyDelta reports no real change', async () => { + const consoleInfoSpy = jest.spyOn(console, 'info').mockImplementation(() => undefined); + noteRepository.getNote.mockResolvedValue(note('a')); + analysisController.applyDelta.mockResolvedValue(null); + + updater.handleNoteChange({ id: 'a', event: 2 }); + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + expect(onGraphPatch).not.toHaveBeenCalled(); + expect(consoleInfoSpy).not.toHaveBeenCalledWith(expect.stringContaining('Incremental update applied')); + consoleInfoSpy.mockRestore(); + }); + + it('requeues a retryable skip and automatically retries after the next coalesce window, with no new event needed', async () => { + noteRepository.getNote.mockImplementation(async (id) => note(id)); + analysisController.applyDelta.mockResolvedValueOnce(null); + analysisController.wasLastDeltaSkippedForRetry.mockReturnValueOnce(true); + + updater.handleNoteChange({ id: 'a', event: 2 }); + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + expect(onGraphPatch).not.toHaveBeenCalled(); + + analysisController.applyDelta.mockResolvedValue({ nodes: [], edges: [] }); + analysisController.wasLastDeltaSkippedForRetry.mockReturnValue(false); + + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + expect(analysisController.applyDelta).toHaveBeenLastCalledWith([note('a')], []); + expect(onGraphPatch).toHaveBeenCalled(); + }); + + it('gives up automatically retrying after 5 consecutive retryable skips, instead of retrying forever', async () => { + const consoleInfoSpy = jest.spyOn(console, 'info').mockImplementation(() => undefined); + noteRepository.getNote.mockImplementation(async (id) => note(id)); + analysisController.applyDelta.mockResolvedValue(null); + analysisController.wasLastDeltaSkippedForRetry.mockReturnValue(true); + + updater.handleNoteChange({ id: 'a', event: 2 }); + for (let i = 0; i < 10; i++) { + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + } + + expect(analysisController.applyDelta).toHaveBeenCalledTimes(5); + expect(consoleInfoSpy).toHaveBeenCalledWith( + expect.stringContaining('Giving up automatic retry after 5 consecutive') + ); + + analysisController.applyDelta.mockResolvedValue({ nodes: [], edges: [] }); + analysisController.wasLastDeltaSkippedForRetry.mockReturnValue(false); + updater.handleNoteChange({ id: 'b', event: 2 }); + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + expect(analysisController.applyDelta).toHaveBeenLastCalledWith( + expect.arrayContaining([note('a'), note('b')]), + [] + ); + consoleInfoSpy.mockRestore(); + }); + + it('folds a note edited again while its retryable skip is still pending into the same retry', async () => { + noteRepository.getNote.mockImplementation(async (id) => note(id)); + analysisController.applyDelta.mockResolvedValueOnce(null); + analysisController.wasLastDeltaSkippedForRetry.mockReturnValueOnce(true); + + updater.handleNoteChange({ id: 'a', event: 2 }); + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + analysisController.applyDelta.mockResolvedValue({ nodes: [], edges: [] }); + analysisController.wasLastDeltaSkippedForRetry.mockReturnValue(false); + + updater.handleNoteChange({ id: 'c', event: 2 }); + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + expect(analysisController.applyDelta).toHaveBeenLastCalledWith( + expect.arrayContaining([note('a'), note('c')]), + [] + ); + }); + + it('does not requeue a delta that applyDelta reports as a non-retryable null (a genuine no-op)', async () => { + noteRepository.getNote.mockImplementation(async (id) => note(id)); + analysisController.applyDelta.mockResolvedValueOnce(null); + analysisController.wasLastDeltaSkippedForRetry.mockReturnValueOnce(false); + + updater.handleNoteChange({ id: 'a', event: 2 }); + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + analysisController.applyDelta.mockResolvedValue({ nodes: [], edges: [] }); + updater.handleNoteChange({ id: 'c', event: 2 }); + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + expect(analysisController.applyDelta).toHaveBeenLastCalledWith([note('c')], []); + }); + + it('does not push a patch if applyDelta succeeds but no diff is available', async () => { + noteRepository.getNote.mockResolvedValue(note('a')); + analysisController.getLastDiff.mockReturnValue(null); + + updater.handleNoteChange({ id: 'a', event: 2 }); + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + expect(onGraphPatch).not.toHaveBeenCalled(); + }); + + it('treats a note that no longer exists by fetch time as a removal, not a dropped upsert', async () => { + noteRepository.getNote.mockResolvedValue(null); + + updater.handleNoteChange({ id: 'a', event: 2 }); + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + expect(analysisController.applyDelta).toHaveBeenCalledWith([], ['a']); + }); + + it('falls back to a full reload if the debounced flush fails to fetch the changed note', async () => { + noteRepository.getNote.mockRejectedValue(new Error('network error')); + + updater.handleNoteChange({ id: 'a', event: 2 }); + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + expect(onFullReloadNeeded).toHaveBeenCalledTimes(1); + expect(analysisController.applyDelta).not.toHaveBeenCalled(); + }); + + it('logs and requeues the delta when the full-reload fallback itself also fails, instead of dropping it silently', async () => { + const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => undefined); + noteRepository.getNote.mockRejectedValue(new Error('network error')); + onFullReloadNeeded.mockRejectedValueOnce(new Error('reload also failed')); + + updater.handleNoteChange({ id: 'a', event: 2 }); + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + expect(consoleErrorSpy).toHaveBeenCalledWith( + 'Full-reload fallback also failed after an incremental flush error:', + expect.any(Error) + ); + + noteRepository.getNote.mockImplementation(async (id) => note(id)); + onFullReloadNeeded.mockResolvedValue(undefined); + updater.handleNoteChange({ id: 'b', event: 2 }); + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + expect(analysisController.applyDelta).toHaveBeenCalledWith( + expect.arrayContaining([note('a'), note('b')]), + [] + ); + consoleErrorSpy.mockRestore(); + }); + + it('does not auto-reschedule after a double failure, but a later sync sweep still picks up the requeued id', async () => { + const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => undefined); + noteRepository.getNote.mockRejectedValue(new Error('network error')); + onFullReloadNeeded.mockRejectedValueOnce(new Error('reload also failed')); + + updater.handleNoteChange({ id: 'a', event: 2 }); + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + jest.clearAllMocks(); + + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS * 5); + expect(analysisController.applyDelta).not.toHaveBeenCalled(); + + noteRepository.getNote.mockImplementation(async (id) => note(id)); + onFullReloadNeeded.mockResolvedValue(undefined); + await updater.handleSyncComplete(); + + expect(analysisController.applyDelta).toHaveBeenCalledWith([note('a')], []); + consoleErrorSpy.mockRestore(); + }); + }); + + describe('handleSelectionChange', () => { + it('schedules an upsert refresh for each selected note', async () => { + noteRepository.getNote.mockImplementation(async (id) => note(id)); + + updater.handleSelectionChange({ value: ['a', 'b'] }); + await jest.advanceTimersByTimeAsync(COALESCE_WINDOW_MS); + + expect(noteRepository.getNote).toHaveBeenCalledWith('a'); + expect(noteRepository.getNote).toHaveBeenCalledWith('b'); + expect(analysisController.applyDelta).toHaveBeenCalledWith( + expect.arrayContaining([note('a'), note('b')]), + [] + ); + }); + }); + + describe('handleSyncComplete (AI off)', () => { + it('does nothing when no note list has been loaded yet', async () => { + analysisController.hasNotes.mockReturnValue(false); + + await updater.handleSyncComplete(); + + expect(eventsRepository.getNoteEventsSince).not.toHaveBeenCalled(); + }); + + it('sweeps with no cursor on the first-ever call and persists the returned baseline', async () => { + eventsRepository.getNoteEventsSince.mockResolvedValue({ events: [], cursor: 'baseline-1' }); + + await updater.handleSyncComplete(); + + expect(eventsRepository.getNoteEventsSince).toHaveBeenCalledWith(undefined); + expect(graphCache.saveEventsCursor).toHaveBeenCalledWith('baseline-1'); + }); + + it('resumes from the persisted cursor on subsequent calls', async () => { + graphCache.loadEventsCursor.mockResolvedValue('cursor-1'); + eventsRepository.getNoteEventsSince.mockResolvedValue({ events: [], cursor: 'cursor-2' }); + + await updater.handleSyncComplete(); + + expect(eventsRepository.getNoteEventsSince).toHaveBeenCalledWith('cursor-1'); + }); + + it('applies created/updated events as upserts and deleted events as removals, then flushes immediately', async () => { + eventsRepository.getNoteEventsSince.mockResolvedValue({ + events: [ + { noteId: 'a', type: 'created' }, + { noteId: 'b', type: 'updated' }, + { noteId: 'c', type: 'deleted' }, + ], + cursor: 'cursor-2', + }); + noteRepository.getNote.mockImplementation(async (id) => note(id)); + + await updater.handleSyncComplete(); + + expect(noteRepository.getNote).toHaveBeenCalledWith('a'); + expect(noteRepository.getNote).toHaveBeenCalledWith('b'); + expect(noteRepository.getNote).not.toHaveBeenCalledWith('c'); + expect(analysisController.applyDelta).toHaveBeenCalledWith( + expect.arrayContaining([note('a'), note('b')]), + ['c'] + ); + }); + + it('falls back to a full reload and does not persist a cursor when the sweep fails', async () => { + eventsRepository.getNoteEventsSince.mockRejectedValue(new Error('network error')); + + await updater.handleSyncComplete(); + + expect(onFullReloadNeeded).toHaveBeenCalledTimes(1); + expect(graphCache.saveEventsCursor).not.toHaveBeenCalled(); + }); + }); + + describe('handleSyncComplete (AI on)', () => { + beforeEach(() => { + checkAiEnabled.mockResolvedValue(true); + }); + + it('schedules upserts from the embeddings cursor sweep, resuming from the persisted cursor', async () => { + graphCache.loadEmbeddingsCursor.mockResolvedValue('embeddings-cursor-1'); + ai.getEmbeddings.mockResolvedValue({ + modelId: 'test-model', + dimension: 2, + chunks: [{ noteId: 'a', vector: [1, 0] }], + nextCursor: undefined, + }); + noteRepository.getNote.mockImplementation(async (id) => note(id)); + + await updater.handleSyncComplete(); + + expect(ai.getEmbeddings).toHaveBeenCalledWith({ cursor: 'embeddings-cursor-1', limit: 1000 }); + expect(graphCache.saveEmbeddingsCursor).toHaveBeenCalledWith('embeddings-cursor-1'); + expect(analysisController.applyDelta).toHaveBeenCalledWith([note('a')], []); + }); + + it('walks multiple embeddings pages, accumulating note ids across them', async () => { + ai.getEmbeddings + .mockResolvedValueOnce({ + modelId: 'test-model', + dimension: 2, + chunks: [{ noteId: 'a', vector: [1, 0] }], + nextCursor: 'page-2', + }) + .mockResolvedValueOnce({ + modelId: 'test-model', + dimension: 2, + chunks: [{ noteId: 'b', vector: [0, 1] }], + nextCursor: undefined, + }); + noteRepository.getNote.mockImplementation(async (id) => note(id)); + + await updater.handleSyncComplete(); + + expect(ai.getEmbeddings).toHaveBeenCalledTimes(2); + expect(ai.getEmbeddings).toHaveBeenNthCalledWith(2, { cursor: 'page-2', limit: 1000 }); + expect(graphCache.saveEmbeddingsCursor).toHaveBeenCalledWith('page-2'); + expect(analysisController.applyDelta).toHaveBeenCalledWith( + expect.arrayContaining([note('a'), note('b')]), + [] + ); + }); + + it('stops at the embeddings page safety cap, saving resumable progress instead of discarding the whole sweep', async () => { + ai.getEmbeddings.mockImplementation(async ({ cursor }) => ({ + modelId: 'test-model', + dimension: 2, + chunks: [{ noteId: `note-${cursor ?? 'start'}`, vector: [1, 0] }], + nextCursor: `next-${cursor ?? 'start'}`, + })); + noteRepository.getNote.mockImplementation(async (id) => note(id)); + + await updater.handleSyncComplete(); + + expect(ai.getEmbeddings).toHaveBeenCalledTimes(500); + expect(onFullReloadNeeded).not.toHaveBeenCalled(); + expect(graphCache.saveEmbeddingsCursor).toHaveBeenCalledWith(expect.any(String)); + expect(analysisController.applyDelta).toHaveBeenCalled(); + }); + + it("only acts on /events' deleted entries, ignoring its created/updated entries since the embeddings sweep already covers those", async () => { + eventsRepository.getNoteEventsSince.mockResolvedValue({ + events: [ + { noteId: 'a', type: 'created' }, + { noteId: 'b', type: 'deleted' }, + ], + cursor: 'events-cursor-2', + }); + + await updater.handleSyncComplete(); + + expect(noteRepository.getNote).not.toHaveBeenCalledWith('a'); + expect(analysisController.applyDelta).toHaveBeenCalledWith([], ['b']); + }); + + it('falls back to /events upserts for this sync when the embeddings sweep fails, without a full reload', async () => { + ai.getIndexStatus.mockResolvedValue({ ready: false, state: 'preparing', modelId: null }); + eventsRepository.getNoteEventsSince.mockResolvedValue({ + events: [{ noteId: 'a', type: 'updated' }], + cursor: 'events-cursor-2', + }); + noteRepository.getNote.mockImplementation(async (id) => note(id)); + + await updater.handleSyncComplete(); + + expect(onFullReloadNeeded).not.toHaveBeenCalled(); + expect(graphCache.saveEmbeddingsCursor).not.toHaveBeenCalled(); + expect(analysisController.applyDelta).toHaveBeenCalledWith([note('a')], []); + }); + + it('still falls back to a full reload if the /events sweep itself also fails', async () => { + ai.getIndexStatus.mockResolvedValue({ ready: false, state: 'preparing', modelId: null }); + eventsRepository.getNoteEventsSince.mockRejectedValue(new Error('network error')); + + await updater.handleSyncComplete(); + + expect(onFullReloadNeeded).toHaveBeenCalledTimes(1); + }); + }); + + describe('flush serialization', () => { + it('never runs two applyDelta calls concurrently when a timer-driven flush overlaps a direct handleSyncComplete flush', async () => { + let concurrentCalls = 0; + let maxConcurrent = 0; + let firstCallPending = true; + let resolveFirstCall: () => void = () => undefined; + + analysisController.applyDelta.mockImplementation(() => { + concurrentCalls++; + maxConcurrent = Math.max(maxConcurrent, concurrentCalls); + if (firstCallPending) { + firstCallPending = false; + return new Promise((resolve) => { + resolveFirstCall = () => { + concurrentCalls--; + resolve({ nodes: [], edges: [] }); + }; + }); + } + concurrentCalls--; + return Promise.resolve({ nodes: [], edges: [] }); + }); + noteRepository.getNote.mockImplementation(async (id) => note(id)); + + updater.handleNoteChange({ id: 'a', event: 1 }); + jest.advanceTimersByTime(COALESCE_WINDOW_MS); + await flushMicrotasks(); + expect(concurrentCalls).toBe(1); + + eventsRepository.getNoteEventsSince.mockResolvedValue({ + events: [{ noteId: 'b', type: 'created' }], + cursor: 'cursor-2', + }); + const syncPromise = updater.handleSyncComplete(); + + await flushMicrotasks(); + expect(concurrentCalls).toBe(1); + + resolveFirstCall(); + await syncPromise; + + expect(maxConcurrent).toBe(1); + expect(analysisController.applyDelta).toHaveBeenCalledTimes(2); + }); + }); +}); diff --git a/src/services/sync/IncrementalUpdater.ts b/src/services/sync/IncrementalUpdater.ts new file mode 100644 index 0000000..830d10a --- /dev/null +++ b/src/services/sync/IncrementalUpdater.ts @@ -0,0 +1,263 @@ +import joplin from 'api'; +import { Note } from '../../data/Types'; +import { NoteRepository } from '../../data/NoteRepository'; +import { NotePreprocessor } from '../../data/NotePreprocessor'; +import { EventsRepository } from '../../data/EventsRepository'; +import { GraphCacheRepository } from '../../data/Database/GraphCacheRepository'; +import { AnalysisController } from '../AnalysisController'; +import { GraphDiff } from '../graph/GraphDiffer'; +import { GraphData } from '../graph/types'; +import { JoplinAiApi, isIndexUsable } from '../embeddings/providers/JoplinNativeProvider'; +import { isAiAnalysisEnabled } from '../settings/GraphSettings'; + +const ITEM_CHANGE_DELETE = 3; + +const EMBEDDINGS_PAGE_SIZE = 1000; +const EMBEDDINGS_MAX_PAGES = 500; +const DEFAULT_COALESCE_WINDOW_MS = 1000; +const MAX_CONSECUTIVE_RETRY_SKIPS = 5; + +export class IncrementalUpdater { + private readonly pendingUpsertIds = new Set(); + private readonly pendingRemovedIds = new Set(); + private flushTimer: ReturnType | null = null; + private flushChain: Promise = Promise.resolve(); + private consecutiveRetrySkips = 0; + + public constructor( + private readonly analysisController: AnalysisController, + private readonly onGraphPatch: (diff: GraphDiff, fullGraphData: GraphData) => void, + private readonly onFullReloadNeeded: () => Promise, + private readonly noteRepository = new NoteRepository(), + private readonly preprocessor = new NotePreprocessor(), + private readonly eventsRepository = new EventsRepository(), + private readonly graphCache = new GraphCacheRepository(), + private readonly coalesceWindowMs = DEFAULT_COALESCE_WINDOW_MS, + private readonly checkAiEnabled: () => Promise = isAiAnalysisEnabled + ) {} + + public handleNoteChange(event: { id: string; event: number }): void { + if (event.event === ITEM_CHANGE_DELETE) { + this.scheduleRemoval(event.id); + } else { + this.scheduleUpsert(event.id); + } + } + + public handleSelectionChange(event: { value: string[] }): void { + for (const id of event.value) { + this.scheduleUpsert(id); + } + } + + public async handleSyncComplete(): Promise { + if (!this.analysisController.hasNotes()) return; + + try { + const aiEnabled = await this.checkAiEnabled(); + let embeddingsSweepFailed = false; + + if (aiEnabled) { + try { + const upsertIds = await this.detectEmbeddingUpserts(); + for (const id of upsertIds) this.scheduleUpsert(id); + } catch (e) { + console.error('Embeddings sweep failed, falling back to /events for this sync:', e); + embeddingsSweepFailed = true; + } + } + + const { upsertIds, removedIds } = await this.detectEventChanges(); + if (!aiEnabled || embeddingsSweepFailed) { + for (const id of upsertIds) this.scheduleUpsert(id); + } + for (const id of removedIds) this.scheduleRemoval(id); + + await this.flush(); + } catch (e) { + console.error('Incremental sync sweep failed, falling back to a full reload:', e); + await this.onFullReloadNeeded(); + } + } + + private async detectEmbeddingUpserts(): Promise { + const cursor = await this.graphCache.loadEmbeddingsCursor(); + const api = this.getAiApi(); + await this.ensureIndexUsable(api); + + const noteIds = new Set(); + let currentCursor = cursor ?? undefined; + let pageCount = 0; + + while (pageCount < EMBEDDINGS_MAX_PAGES) { + pageCount++; + + const page = await api.getEmbeddings({ cursor: currentCursor, limit: EMBEDDINGS_PAGE_SIZE }); + for (const chunk of page.chunks) { + noteIds.add(chunk.noteId); + } + + if (!page.nextCursor) break; + currentCursor = page.nextCursor; + } + + if (pageCount >= EMBEDDINGS_MAX_PAGES) { + console.info( + `Embeddings sweep hit the ${EMBEDDINGS_MAX_PAGES}-page safety cap; remaining changes will be picked up on the next sync.` + ); + } + + if (currentCursor) { + await this.graphCache.saveEmbeddingsCursor(currentCursor); + } + + return Array.from(noteIds); + } + + private getAiApi(): JoplinAiApi { + const api = joplin.ai as unknown as JoplinAiApi | undefined; + if (!api) { + throw new Error('joplin.ai is not available. Enable AI in Settings → AI.'); + } + return api; + } + + private async ensureIndexUsable(api: JoplinAiApi): Promise { + const status = await api.getIndexStatus(); + if (!status || !isIndexUsable(status.state)) { + throw new Error( + `Joplin AI index is not usable yet (state: ${status?.state ?? 'unknown'}). ` + + 'Enable AI and wait for the embedding model to finish loading in Settings → AI.' + ); + } + } + + private async detectEventChanges(): Promise<{ upsertIds: string[]; removedIds: string[] }> { + const cursor = await this.graphCache.loadEventsCursor(); + const { events, cursor: nextCursor } = await this.eventsRepository.getNoteEventsSince( + cursor ?? undefined + ); + + const upsertIds: string[] = []; + const removedIds: string[] = []; + for (const event of events) { + if (event.type === 'deleted') { + removedIds.push(event.noteId); + } else { + upsertIds.push(event.noteId); + } + } + + if (nextCursor) { + await this.graphCache.saveEventsCursor(nextCursor); + } + + return { upsertIds, removedIds }; + } + + private scheduleUpsert(id: string): void { + this.pendingRemovedIds.delete(id); + this.pendingUpsertIds.add(id); + this.scheduleFlush(); + } + + private scheduleRemoval(id: string): void { + this.pendingUpsertIds.delete(id); + this.pendingRemovedIds.add(id); + this.scheduleFlush(); + } + + private scheduleFlush(): void { + if (this.flushTimer) return; + this.flushTimer = setTimeout(() => { + this.flushTimer = null; + void this.flush(); + }, this.coalesceWindowMs); + } + + private flush(): Promise { + const task = this.flushChain.then(() => this.flushInternal()); + this.flushChain = task.then( + () => undefined, + () => undefined + ); + return task; + } + + private async flushInternal(): Promise { + if (this.flushTimer) { + clearTimeout(this.flushTimer); + this.flushTimer = null; + } + + const upsertIds = Array.from(this.pendingUpsertIds); + const removedIds = Array.from(this.pendingRemovedIds); + this.pendingUpsertIds.clear(); + this.pendingRemovedIds.clear(); + + if (upsertIds.length === 0 && removedIds.length === 0) return; + if (!this.analysisController.hasNotes()) return; + + try { + const { upserts, discoveredRemovals } = await this.fetchAndEnrich(upsertIds); + const graphData = await this.analysisController.applyDelta(upserts, [ + ...removedIds, + ...discoveredRemovals, + ]); + if (!graphData) { + if (this.analysisController.wasLastDeltaSkippedForRetry()) { + for (const id of upsertIds) this.pendingUpsertIds.add(id); + for (const id of removedIds) this.pendingRemovedIds.add(id); + this.consecutiveRetrySkips++; + if (this.consecutiveRetrySkips < MAX_CONSECUTIVE_RETRY_SKIPS) { + this.scheduleFlush(); + } else { + console.info( + `Giving up automatic retry after ${this.consecutiveRetrySkips} consecutive skipped updates; will retry on the next edit or sync.` + ); + } + } else { + this.consecutiveRetrySkips = 0; + } + return; + } + + this.consecutiveRetrySkips = 0; + const removedCount = removedIds.length + discoveredRemovals.length; + console.info(`Incremental update applied: ${upserts.length} upserted, ${removedCount} removed.`); + + const diff = this.analysisController.getLastDiff(); + if (diff) { + this.onGraphPatch(diff, graphData); + } + } catch (e) { + this.consecutiveRetrySkips = 0; + console.error('Incremental flush failed, falling back to a full reload:', e); + try { + await this.onFullReloadNeeded(); + } catch (fallbackError) { + console.error('Full-reload fallback also failed after an incremental flush error:', fallbackError); + for (const id of upsertIds) this.pendingUpsertIds.add(id); + for (const id of removedIds) this.pendingRemovedIds.add(id); + } + } + } + + private async fetchAndEnrich( + ids: string[] + ): Promise<{ upserts: Note[]; discoveredRemovals: string[] }> { + const upserts: Note[] = []; + const discoveredRemovals: string[] = []; + + for (const id of ids) { + const raw = await this.noteRepository.getNote(id); + if (!raw) { + discoveredRemovals.push(id); + continue; + } + upserts.push(await this.preprocessor.processOne(raw)); + } + + return { upserts, discoveredRemovals }; + } +} diff --git a/src/services/sync/WorkspaceListener.test.ts b/src/services/sync/WorkspaceListener.test.ts new file mode 100644 index 0000000..aa3f3cd --- /dev/null +++ b/src/services/sync/WorkspaceListener.test.ts @@ -0,0 +1,61 @@ +import joplin from 'api'; +import { WorkspaceListener } from './WorkspaceListener'; +import { IncrementalUpdater } from './IncrementalUpdater'; + +jest.mock('./IncrementalUpdater'); + +const MockIncrementalUpdater = IncrementalUpdater as jest.MockedClass; + +describe('WorkspaceListener', () => { + it('registers all three workspace events and forwards them to the matching updater method', async () => { + const updater = new MockIncrementalUpdater( + {} as never, + {} as never, + jest.fn() + ) as jest.Mocked; + updater.handleSyncComplete.mockResolvedValue(undefined); + const listener = new WorkspaceListener(updater); + + await listener.register(); + + expect(joplin.workspace.onNoteChange).toHaveBeenCalledTimes(1); + expect(joplin.workspace.onNoteSelectionChange).toHaveBeenCalledTimes(1); + expect(joplin.workspace.onSyncComplete).toHaveBeenCalledTimes(1); + + const noteChangeCallback = (joplin.workspace.onNoteChange as jest.Mock).mock.calls[0][0]; + noteChangeCallback({ id: 'a', event: 1 }); + expect(updater.handleNoteChange).toHaveBeenCalledWith({ id: 'a', event: 1 }); + + const selectionCallback = (joplin.workspace.onNoteSelectionChange as jest.Mock).mock + .calls[0][0]; + selectionCallback({ value: ['a'] }); + expect(updater.handleSelectionChange).toHaveBeenCalledWith({ value: ['a'] }); + + const syncCompleteCallback = (joplin.workspace.onSyncComplete as jest.Mock).mock.calls[0][0]; + syncCompleteCallback(); + expect(updater.handleSyncComplete).toHaveBeenCalledTimes(1); + }); + + it('does not let a handleSyncComplete rejection become an unhandled rejection', async () => { + const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => undefined); + const updater = new MockIncrementalUpdater( + {} as never, + {} as never, + jest.fn() + ) as jest.Mocked; + updater.handleSyncComplete.mockRejectedValue(new Error('full reload also failed')); + const listener = new WorkspaceListener(updater); + await listener.register(); + + const syncCompleteCallback = (joplin.workspace.onSyncComplete as jest.Mock).mock.calls[0][0]; + syncCompleteCallback(); + await Promise.resolve(); + await Promise.resolve(); + + expect(consoleErrorSpy).toHaveBeenCalledWith( + 'Sync-complete handling failed:', + expect.any(Error) + ); + consoleErrorSpy.mockRestore(); + }); +}); diff --git a/src/services/sync/WorkspaceListener.ts b/src/services/sync/WorkspaceListener.ts new file mode 100644 index 0000000..c3f9668 --- /dev/null +++ b/src/services/sync/WorkspaceListener.ts @@ -0,0 +1,18 @@ +import joplin from 'api'; +import { IncrementalUpdater } from './IncrementalUpdater'; + +export class WorkspaceListener { + public constructor(private readonly updater: IncrementalUpdater) {} + + public async register(): Promise { + await joplin.workspace.onNoteChange((event) => this.updater.handleNoteChange(event)); + await joplin.workspace.onNoteSelectionChange((event) => + this.updater.handleSelectionChange(event) + ); + await joplin.workspace.onSyncComplete(() => { + this.updater.handleSyncComplete().catch((e) => { + console.error('Sync-complete handling failed:', e); + }); + }); + } +} diff --git a/src/tests/mocks/joplin.ts b/src/tests/mocks/joplin.ts index a505703..8d20bb9 100644 --- a/src/tests/mocks/joplin.ts +++ b/src/tests/mocks/joplin.ts @@ -14,12 +14,37 @@ const joplinSettings = { onChange: jest.fn(), }; +const joplinWorkspace = { + onNoteChange: jest.fn(), + onNoteSelectionChange: jest.fn(), + onSyncComplete: jest.fn(), +}; + +const joplinViewsPanels = { + create: jest.fn(), + setHtml: jest.fn(), + onMessage: jest.fn(), + addScript: jest.fn(), + show: jest.fn(), + hide: jest.fn(), + postMessage: jest.fn(), +}; + +const joplinCommands = { + execute: jest.fn(), +}; + const joplin = { data: { get: jest.fn(), }, ai: joplinAi, settings: joplinSettings, + workspace: joplinWorkspace, + views: { + panels: joplinViewsPanels, + }, + commands: joplinCommands, }; export default joplin; diff --git a/src/ui/graph-view.js b/src/ui/graph-view.js index 8683aa1..b105c65 100644 --- a/src/ui/graph-view.js +++ b/src/ui/graph-view.js @@ -29,14 +29,22 @@ var FCOSE_OPTIONS = { step: 'all', }; +var INCREMENTAL_FCOSE_OVERRIDES = { + randomize: false, + animate: false, + fit: false, + packComponents: false, +}; + var cy; var statusEl; -var pollTimer; var tooltipEl; var nodeStats; var progressEl; var progressFillEl; var progressLabelEl; +var hasRenderedOnce = false; +var lastSeenVersion = 0; function showStatus(text) { if (statusEl) { @@ -196,33 +204,14 @@ function onNodeDblClick(evt) { }); } -/** - * Replace the current graph with new data. Computes per-node link/tag counts, - * deduplicates unique tag names for the stats bar, and runs the fCoSE layout. - * @param {{ nodes: Array, edges: Array }} message - graph data from the plugin. - */ -function renderGraph(message) { - cy.elements().remove(); - - if (!message || !message.nodes || !message.nodes.length) { - showStatus('No graph data received'); - updateStats(0, 0, 0, 0); - return; - } - - hideStatus(); - - cy.add(message.nodes); - cy.add(message.edges || []); - +function recomputeStats() { nodeStats = {}; - var edgesArr = message.edges || []; var explicitCount = 0; var semanticCount = 0; var tagNames = {}; - for (var i = 0; i < edgesArr.length; i++) { - var e = edgesArr[i].data || edgesArr[i]; + cy.edges().forEach(function (edge) { + var e = edge.data(); if (!nodeStats[e.source]) nodeStats[e.source] = { linkCount: 0, tagCount: 0 }; if (!nodeStats[e.target]) nodeStats[e.target] = { linkCount: 0, tagCount: 0 }; @@ -244,19 +233,171 @@ function renderGraph(message) { } } } - } + }); var totalTags = Object.keys(tagNames).length; - updateStats(message.nodes.length, explicitCount, semanticCount, totalTags); + updateStats(cy.nodes().length, explicitCount, semanticCount, totalTags); +} + +function refreshEmptyStateStatus() { + if (cy.nodes().length === 0) { + showStatus('No graph data received'); + } else if (cy.edges().length === 0) { + showStatus(cy.nodes().length + ' notes, 0 connections'); + } else { + hideStatus(); + } +} + +/** + * Replace the current graph with new data and run a full fCoSE layout. + * @param {{ nodes: Array, edges: Array }} message - graph data from the plugin. + */ +function renderGraph(message) { + cy.elements().remove(); + if (!message || !message.nodes || !message.nodes.length) { + showStatus('No graph data received'); + updateStats(0, 0, 0, 0); + return; + } + + hideStatus(); + + cy.add(message.nodes); + cy.add(message.edges || []); + + recomputeStats(); cy.layout(FCOSE_OPTIONS).run(); + refreshEmptyStateStatus(); +} - var edgeCount = (message.edges || []).length; - if (edgeCount === 0) { - showStatus(message.nodes.length + ' notes, 0 connections'); +function upsertElement(data) { + var existing = cy.getElementById(data.id); + if (existing && existing.length) { + existing.data(data); } else { - hideStatus(); + cy.add({ data: data }); + } +} + +function applyGraphPatch(patch) { + if (!cy || !patch) return; + var hasChanges = + (patch.upsertedNodes && patch.upsertedNodes.length) || + (patch.upsertedEdges && patch.upsertedEdges.length) || + (patch.removedNodeIds && patch.removedNodeIds.length) || + (patch.removedEdgeIds && patch.removedEdgeIds.length); + if (!hasChanges) return; + + var movableIds = {}; + + (patch.removedEdgeIds || []).forEach(function (id) { + var ele = cy.getElementById(id); + if (ele && ele.length) { + movableIds[ele.data('source')] = true; + movableIds[ele.data('target')] = true; + } + }); + + var toRemove = cy.collection(); + (patch.removedEdgeIds || []).concat(patch.removedNodeIds || []).forEach(function (id) { + var ele = cy.getElementById(id); + if (ele && ele.length) toRemove = toRemove.union(ele); + }); + toRemove.remove(); + + (patch.upsertedNodes || []).forEach(function (item) { + var data = item.data || item; + var existing = cy.getElementById(data.id); + var isNew = !(existing && existing.length); + upsertElement(data); + if (isNew) movableIds[data.id] = true; + }); + (patch.upsertedEdges || []).forEach(function (item) { + var data = item.data || item; + upsertElement(data); + movableIds[data.source] = true; + movableIds[data.target] = true; + }); + + recomputeStats(); + + var fixedNodeConstraint = []; + cy.nodes().forEach(function (n) { + if (!movableIds[n.id()]) { + fixedNodeConstraint.push({ nodeId: n.id(), position: n.position() }); + } + }); + cy.layout( + Object.assign({}, FCOSE_OPTIONS, INCREMENTAL_FCOSE_OVERRIDES, { + fixedNodeConstraint: fixedNodeConstraint, + }) + ).run(); + + refreshEmptyStateStatus(); +} + +function dataEqual(existingEle, data) { + if (!existingEle || !existingEle.length) return false; + var existing = existingEle.data(); + var existingKeys = Object.keys(existing); + var newKeys = Object.keys(data); + if (existingKeys.length !== newKeys.length) return false; + return existingKeys.every(function (key) { + return existing[key] === data[key]; + }); +} + +function computeClientPatch(graphData) { + var newNodeIds = {}; + (graphData.nodes || []).forEach(function (n) { + newNodeIds[n.data.id] = true; + }); + var newEdgeIds = {}; + (graphData.edges || []).forEach(function (e) { + newEdgeIds[e.data.id] = true; + }); + + var upsertedNodes = (graphData.nodes || []).filter(function (n) { + return !dataEqual(cy.getElementById(n.data.id), n.data); + }); + var upsertedEdges = (graphData.edges || []).filter(function (e) { + return !dataEqual(cy.getElementById(e.data.id), e.data); + }); + + var removedNodeIds = []; + cy.nodes().forEach(function (n) { + if (!newNodeIds[n.id()]) removedNodeIds.push(n.id()); + }); + var removedEdgeIds = []; + cy.edges().forEach(function (e) { + if (!newEdgeIds[e.id()]) removedEdgeIds.push(e.id()); + }); + + return { + upsertedNodes: upsertedNodes, + upsertedEdges: upsertedEdges, + removedNodeIds: removedNodeIds, + removedEdgeIds: removedEdgeIds, + }; +} + +function handleGraphUpdate(type, message) { + var version = message.version || 0; + if (hasRenderedOnce && version <= lastSeenVersion) return; + + if (type === 'graph-patch') { + if (!hasRenderedOnce || version !== lastSeenVersion + 1) return; + applyGraphPatch(message); + } else if (!hasRenderedOnce) { + renderGraph(message); + hasRenderedOnce = true; + } else { + applyGraphPatch(computeClientPatch(message)); } + + lastSeenVersion = version; } /** Write counts into the stats bar elements (stat-notes, stat-explicit, stat-semantic, stat-tags). */ @@ -324,20 +465,31 @@ function downloadFile(data, filename) { if (data.indexOf('blob:') === 0) URL.revokeObjectURL(data); } -/** Poll every second for graph data via webviewApi until the first graph-data response arrives. */ -function pollForData() { - if (typeof webviewApi === 'undefined') { - return; - } +var POLL_INTERVAL_WAITING_MS = 1000; +var POLL_INTERVAL_LIVE_MS = 3000; - pollTimer = setInterval(function () { - webviewApi.postMessage({ type: 'request-data' }).then(function (response) { +function requestData() { + webviewApi + .postMessage({ type: 'request-data', version: lastSeenVersion }) + .then(function (response) { if (response && response.type === 'graph-data') { - clearInterval(pollTimer); - renderGraph(response); + hideProgress(); + handleGraphUpdate('graph-data', response); } + }) + .catch(function (e) { + console.error('Note Graph poll failed:', e); + }) + .then(function () { + setTimeout(requestData, hasRenderedOnce ? POLL_INTERVAL_LIVE_MS : POLL_INTERVAL_WAITING_MS); }); - }, 1000); +} + +function pollForData() { + if (typeof webviewApi === 'undefined') { + return; + } + requestData(); } /** @@ -563,12 +715,12 @@ function init() { if (typeof webviewApi !== 'undefined') { webviewApi.onMessage(function (message) { if (message && message.type === 'graph-data') { - if (pollTimer) { - clearInterval(pollTimer); - pollTimer = null; - } hideProgress(); - renderGraph(message); + handleGraphUpdate('graph-data', message); + } + if (message && message.type === 'graph-patch') { + hideProgress(); + handleGraphUpdate('graph-patch', message); } if (message && message.type === 'fit-to-screen') { cy.fit(undefined, 30); @@ -583,6 +735,7 @@ function init() { }); } } catch (e) { + console.error('Note Graph panel failed to initialize:', e); showStatus('Error: ' + (e && e.message ? e.message : String(e))); } } diff --git a/src/ui/webview.test.ts b/src/ui/webview.test.ts new file mode 100644 index 0000000..3d6dcff --- /dev/null +++ b/src/ui/webview.test.ts @@ -0,0 +1,75 @@ +import { GraphDiff } from '../services/graph/GraphDiffer'; + +const emptyDiff: GraphDiff = { + upsertedNodes: [], + upsertedEdges: [], + removedNodeIds: [], + removedEdgeIds: [], +}; + +describe('webview', () => { + let webview: typeof import('./webview'); + let mockPanelsCreate: jest.Mock; + let mockOnMessage: jest.Mock; + let mockPostMessage: jest.Mock; + let onMessageHandler: (message: { type?: string; version?: number }) => Promise; + + beforeEach(async () => { + jest.resetModules(); + + let freshJoplin: { + views: { panels: { create: jest.Mock; onMessage: jest.Mock; postMessage: jest.Mock } }; + }; + jest.isolateModules(() => { + // eslint-disable-next-line @typescript-eslint/no-var-requires + freshJoplin = require('api').default; + // eslint-disable-next-line @typescript-eslint/no-var-requires + webview = require('./webview'); + }); + + mockPanelsCreate = freshJoplin!.views.panels.create; + mockOnMessage = freshJoplin!.views.panels.onMessage; + mockPostMessage = freshJoplin!.views.panels.postMessage; + + mockPanelsCreate.mockResolvedValue('panel-handle'); + mockOnMessage.mockImplementation((_handle: unknown, handler: typeof onMessageHandler) => { + onMessageHandler = handler; + return Promise.resolve(); + }); + + await webview.initializeAiNoteGraphPanel(); + }); + + it('replies no-data to request-data before any graph has been loaded', async () => { + const response = await onMessageHandler({ type: 'request-data', version: 0 }); + expect(response).toEqual({ type: 'no-data' }); + }); + + it('replies with the full graph, including the version field, when the requester is behind', async () => { + await webview.postGraphData({ nodes: [], edges: [] }); + + const response = await onMessageHandler({ type: 'request-data', version: 0 }); + + expect(response).toEqual({ type: 'graph-data', nodes: [], edges: [], version: 1 }); + }); + + it('replies no-change instead of re-sending the graph when the requester is already current', async () => { + await webview.postGraphData({ nodes: [], edges: [] }); + + const response = await onMessageHandler({ type: 'request-data', version: 1 }); + + expect(response).toEqual({ type: 'no-change' }); + }); + + it('keeps postGraphData and postGraphPatch on one shared, contiguous version counter', async () => { + await webview.postGraphData({ nodes: [], edges: [] }); + await webview.postGraphPatch(emptyDiff, { nodes: [], edges: [] }); + await webview.postGraphData({ nodes: [], edges: [] }); + + const pushedVersions = mockPostMessage.mock.calls.map( + ([, message]: [unknown, { version: number }]) => message.version + ); + + expect(pushedVersions).toEqual([2, 3]); + }); +}); diff --git a/src/ui/webview.ts b/src/ui/webview.ts index f94c317..0497aa4 100644 --- a/src/ui/webview.ts +++ b/src/ui/webview.ts @@ -2,6 +2,7 @@ import joplin from 'api'; import { ViewHandle } from 'api/types'; import { renderPanelHtml } from './App'; import { GraphData } from '../services/graph/types'; +import { GraphDiff } from '../services/graph/GraphDiffer'; const PANEL_ID = 'aiNoteGraphPanel'; const PANEL_HTML = renderPanelHtml(); @@ -9,22 +10,26 @@ const PANEL_SCRIPTS = ['./ui/styles/panel.css', './ui/setup.js', './ui/graph-vie let panelHandle: ViewHandle; let currentGraphData: GraphData | null = null; +let currentVersion = 0; const createPanel = async (): Promise => { const handle = await joplin.views.panels.create(PANEL_ID); await joplin.views.panels.setHtml(handle, PANEL_HTML); await joplin.views.panels.onMessage( handle, - async (message: { type?: string; nodeId?: string; nodeLabel?: string }) => { + async (message: { type?: string; nodeId?: string; nodeLabel?: string; version?: number }) => { if (message?.type === 'close-note-graph') { await joplin.views.panels.hide(handle); return { done: true }; } if (message?.type === 'request-data') { - if (currentGraphData) { - return { type: 'graph-data', ...currentGraphData }; + if (!currentGraphData) { + return { type: 'no-data' }; } - return { type: 'no-data' }; + if (message.version === currentVersion) { + return { type: 'no-change' }; + } + return { type: 'graph-data', ...currentGraphData, version: currentVersion }; } if (message?.type === 'node-clicked' && message?.nodeId) { try { @@ -78,10 +83,30 @@ export const showAiNoteGraphPanel = async (): Promise => { export const postGraphData = async (graphData: GraphData): Promise => { const hadData = currentGraphData !== null; currentGraphData = graphData; + currentVersion++; + + if (hadData) { + const handle = getPanel(); + joplin.views.panels.postMessage(handle, { + type: 'graph-data', + ...graphData, + version: currentVersion, + }); + } +}; + +export const postGraphPatch = async (diff: GraphDiff, fullGraphData: GraphData): Promise => { + const hadData = currentGraphData !== null; + currentGraphData = fullGraphData; + currentVersion++; if (hadData) { const handle = getPanel(); - joplin.views.panels.postMessage(handle, { type: 'graph-data', ...graphData }); + joplin.views.panels.postMessage(handle, { + type: 'graph-patch', + ...diff, + version: currentVersion, + }); } };