hlassiege/panoplie · 7fa277e

modified packages/core/src/catalog/index.ts +43 -41 Viewed

@@ -18,7 +18,7 @@ import {
18	18	} from '../catalog-model.ts'
19	19	import type { CatalogStore, OpenCatalog } from '../contracts.ts'
20	20	import type { HookDetails, InventoryItem, McpDetails } from '../model.ts'
21		-import { hashPath, isDirectory, listDirs, listFilesRecursive, parseFrontmatter, pathExists, readText } from '../util.ts'
	21	+import { hashPath, isDirectory, listDirs, listFilesRecursive, parseFrontmatter, pathExists, pathFingerprint, readText } from '../util.ts'
22	22	import { atomicWrite, copyDir, movePath, tmpSuffix } from './fsutil.ts'
23	23	import { assertValidName, deriveName, isInside, normalizeTags, resolveInside } from './names.ts'
24	24	import { normalizeSkillMd, parseHookYaml, parseMcpYaml, parseYamlDoc } from './templates.ts'
@@ -137,19 +137,6 @@ class Catalog implements CatalogStore {
137	137
138	138	// ---- reading ----
139	139
140		- private async newestMtime(p: string): Promise<string> {
141		- let newest = 0
142		- const files = (await isDirectory(p)) ? (await listFilesRecursive(p, 2000)).map((f) => path.join(p, f)) : [p]
143		- for (const f of files) {
144		- try {
145		- newest = Math.max(newest, (await stat(f)).mtimeMs)
146		- } catch {
147		- // ignore
148		- }
149		- }
150		- return new Date(newest \|\| Date.now()).toISOString()
151		- }
152		-
153	140	private async buildItem(
154	141	ref: CatalogRef,
155	142	description: string \| undefined,
@@ -157,13 +144,14 @@ class Catalog implements CatalogStore {
157	144	): Promise<CatalogItem> {
158	145	const p = this.itemPath(ref)
159	146	const entry = index.items[itemKey(ref)]
	147	+ const { hash, newestMtimeMs } = await pathFingerprint(p)
160	148	const item: CatalogItem = {
161	149	type: ref.type,
162	150	name: ref.name,
163	151	tags: entry?.tags ?? [],
164	152	path: p,
165		- hash: (await hashPath(p)) ?? '',
166		- updatedAt: await this.newestMtime(p),
	153	+ hash: hash ?? '',
	154	+ updatedAt: new Date(newestMtimeMs \|\| Date.now()).toISOString(),
167	155	}
168	156	if (description !== undefined) item.description = description
169	157	if (entry?.origin) item.origin = entry.origin
@@ -197,17 +185,25 @@ class Catalog implements CatalogStore {
197	185	names = []
198	186	}
199	187	}
200		- for (const name of names) {
201		- if (name.startsWith('.')) continue // staging folders, dotfiles
202		- const ref = { type, name }
203		- try {
204		- assertValidName(name, `${type} name`)
205		- const content = await readText(this.mainFile(ref))
206		- if (content === undefined) throw new Error(`Missing ${type === 'skill' ? 'SKILL.md' : 'file'}`)
207		- snap.items.push(await this.buildItem(ref, describeContent(type, content, name), index))
208		- } catch (e) {
209		- fail(path.join(dir, name), e)
210		- }
	188	+ // Items are read in parallel (hashing every file one by one is slow on Windows); errors keep the folder order.
	189	+ const results = await Promise.all(
	190	+ names
	191	+ .filter((name) => !name.startsWith('.')) // staging folders, dotfiles
	192	+ .map(async (name): Promise<{ item: CatalogItem } \| { name: string; error: unknown }> => {
	193	+ const ref = { type, name }
	194	+ try {
	195	+ assertValidName(name, `${type} name`)
	196	+ const content = await readText(this.mainFile(ref))
	197	+ if (content === undefined) throw new Error(`Missing ${type === 'skill' ? 'SKILL.md' : 'file'}`)
	198	+ return { item: await this.buildItem(ref, describeContent(type, content, name), index) }
	199	+ } catch (error) {
	200	+ return { name, error }
	201	+ }
	202	+ }),
	203	+ )
	204	+ for (const r of results) {
	205	+ if ('item' in r) snap.items.push(r.item)
	206	+ else fail(path.join(dir, r.name), r.error)
211	207	}
212	208	}
213	209	let presetFiles: string[] = []
@@ -260,15 +256,18 @@ class Catalog implements CatalogStore {
260	256	return (await pathExists(p)) ? [{ rel: path.basename(p), abs: p }] : []
261	257	}
262	258
263		- async getItem(ref: CatalogRef): Promise<CatalogItemDetail \| undefined> {
	259	+ async getItem(ref: CatalogRef, known?: CatalogItem): Promise<CatalogItemDetail \| undefined> {
264	260	const main = this.mainFile(ref)
265	261	const content = await readText(main)
266	262	if (content === undefined) return undefined
	263	+ const reuse = known && known.type === ref.type && known.name === ref.name ? known : undefined
267	264	let index: CatalogIndex = { version: 1, items: {} }
268		- try {
269		- index = await this.readIndex()
270		- } catch {
271		- // tags unavailable
	265	+ if (!reuse) {
	266	+ try {
	267	+ index = await this.readIndex()
	268	+ } catch {
	269	+ // tags unavailable
	270	+ }
272	271	}
273	272	let description: string \| undefined
274	273	let template: McpTemplate \| HookTemplate \| undefined
@@ -279,15 +278,18 @@ class Catalog implements CatalogStore {
279	278	} catch {
280	279	// invalid content: still return it so it can be fixed
281	280	}
282		- const files: CatalogFile[] = []
283		- for (const f of await this.itemFiles(ref)) {
284		- try {
285		- files.push({ path: f.rel, sizeBytes: (await stat(f.abs)).size })
286		- } catch {
287		- // vanished
288		- }
289		- }
290		- const detail: CatalogItemDetail = { item: await this.buildItem(ref, description, index), content, files }
	281	+ const sizes = await Promise.all(
	282	+ (await this.itemFiles(ref)).map(async (f): Promise<CatalogFile \| undefined> => {
	283	+ try {
	284	+ return { path: f.rel, sizeBytes: (await stat(f.abs)).size }
	285	+ } catch {
	286	+ return undefined // vanished
	287	+ }
	288	+ }),
	289	+ )
	290	+ const files = sizes.filter((f): f is CatalogFile => f !== undefined)
	291	+ const item = reuse ?? (await this.buildItem(ref, description, index))
	292	+ const detail: CatalogItemDetail = { item, content, files }
291	293	if (template) detail.template = template
292	294	return detail
293	295	}

  } from '../catalog-model.ts'
  import type { CatalogStore, OpenCatalog } from '../contracts.ts'
  import type { HookDetails, InventoryItem, McpDetails } from '../model.ts'
--import { hashPath, isDirectory, listDirs, listFilesRecursive, parseFrontmatter, pathExists, readText } from '../util.ts'
++import { hashPath, isDirectory, listDirs, listFilesRecursive, parseFrontmatter, pathExists, pathFingerprint, readText } from '../util.ts'
  import { atomicWrite, copyDir, movePath, tmpSuffix } from './fsutil.ts'
  import { assertValidName, deriveName, isInside, normalizeTags, resolveInside } from './names.ts'
  import { normalizeSkillMd, parseHookYaml, parseMcpYaml, parseYamlDoc } from './templates.ts'
    // ---- reading ----
--  private async newestMtime(p: string): Promise<string> {
--    let newest = 0
--    const files = (await isDirectory(p)) ? (await listFilesRecursive(p, 2000)).map((f) => path.join(p, f)) : [p]
--    for (const f of files) {
--      try {
--        newest = Math.max(newest, (await stat(f)).mtimeMs)
--      } catch {
--        // ignore
--      }
--    }
--    return new Date(newest || Date.now()).toISOString()
--  }
--
    private async buildItem(
      ref: CatalogRef,
      description: string | undefined,
    ): Promise<CatalogItem> {
      const p = this.itemPath(ref)
      const entry = index.items[itemKey(ref)]
++    const { hash, newestMtimeMs } = await pathFingerprint(p)
      const item: CatalogItem = {
        type: ref.type,
        name: ref.name,
        tags: entry?.tags ?? [],
        path: p,
--      hash: (await hashPath(p)) ?? '',
++      hash: hash ?? '',
--      updatedAt: await this.newestMtime(p),
++      updatedAt: new Date(newestMtimeMs || Date.now()).toISOString(),
      }
      if (description !== undefined) item.description = description
      if (entry?.origin) item.origin = entry.origin
            names = []
          }
        }
--      for (const name of names) {
++      // Items are read in parallel (hashing every file one by one is slow on Windows); errors keep the folder order.
--        if (name.startsWith('.')) continue // staging folders, dotfiles
++      const results = await Promise.all(
--        const ref = { type, name }
++        names
--        try {
++          .filter((name) => !name.startsWith('.')) // staging folders, dotfiles
--          assertValidName(name, `${type} name`)
++          .map(async (name): Promise<{ item: CatalogItem } | { name: string; error: unknown }> => {
--          const content = await readText(this.mainFile(ref))
++            const ref = { type, name }
--          if (content === undefined) throw new Error(`Missing ${type === 'skill' ? 'SKILL.md' : 'file'}`)
++            try {
--          snap.items.push(await this.buildItem(ref, describeContent(type, content, name), index))
++              assertValidName(name, `${type} name`)
--        } catch (e) {
++              const content = await readText(this.mainFile(ref))
--          fail(path.join(dir, name), e)
++              if (content === undefined) throw new Error(`Missing ${type === 'skill' ? 'SKILL.md' : 'file'}`)
--        }
++              return { item: await this.buildItem(ref, describeContent(type, content, name), index) }
++            } catch (error) {
++              return { name, error }
++            }
++          }),
++      )
++      for (const r of results) {
++        if ('item' in r) snap.items.push(r.item)
++        else fail(path.join(dir, r.name), r.error)
        }
      }
      let presetFiles: string[] = []
      return (await pathExists(p)) ? [{ rel: path.basename(p), abs: p }] : []
    }
--  async getItem(ref: CatalogRef): Promise<CatalogItemDetail | undefined> {
++  async getItem(ref: CatalogRef, known?: CatalogItem): Promise<CatalogItemDetail | undefined> {
      const main = this.mainFile(ref)
      const content = await readText(main)
      if (content === undefined) return undefined
++    const reuse = known && known.type === ref.type && known.name === ref.name ? known : undefined
      let index: CatalogIndex = { version: 1, items: {} }
--    try {
++    if (!reuse) {
--      index = await this.readIndex()
++      try {
--    } catch {
++        index = await this.readIndex()
--      // tags unavailable
++      } catch {
++        // tags unavailable
++      }
      }
      let description: string | undefined
      let template: McpTemplate | HookTemplate | undefined
      } catch {
        // invalid content: still return it so it can be fixed
      }
--    const files: CatalogFile[] = []
++    const sizes = await Promise.all(
--    for (const f of await this.itemFiles(ref)) {
++      (await this.itemFiles(ref)).map(async (f): Promise<CatalogFile | undefined> => {
--      try {
++        try {
--        files.push({ path: f.rel, sizeBytes: (await stat(f.abs)).size })
++          return { path: f.rel, sizeBytes: (await stat(f.abs)).size }
--      } catch {
++        } catch {
--        // vanished
++          return undefined // vanished
--      }
++        }
--    }
++      }),
--    const detail: CatalogItemDetail = { item: await this.buildItem(ref, description, index), content, files }
++    )
++    const files = sizes.filter((f): f is CatalogFile => f !== undefined)
++    const item = reuse ?? (await this.buildItem(ref, description, index))
++    const detail: CatalogItemDetail = { item, content, files }
      if (template) detail.template = template
      return detail
    }

modified packages/core/src/contracts.ts +2 -1 Viewed

@@ -31,7 +31,8 @@ export interface CatalogStore {
31	31	/** Creates the folder layout and `.skilly/index.json` if missing (idempotent). Does not run git. */
32	32	init(): Promise<void>
33	33	snapshot(): Promise<CatalogSnapshot>
34		- getItem(ref: CatalogRef): Promise<CatalogItemDetail \| undefined>
	34	+ /** `known`: the item as listed by a fresh snapshot, reused instead of hashing the item again. */
	35	+ getItem(ref: CatalogRef, known?: CatalogItem): Promise<CatalogItemDetail \| undefined>
35	36	/** Absolute paths of all files of an item (folder content for skills, the single file otherwise). */
36	37	itemFiles(ref: CatalogRef): Promise<{ rel: string; abs: string }[]>
37	38	/** Create or update (or rename with `previousName`). Validates names and mcp/hook YAML. */

    /** Creates the folder layout and `.skilly/index.json` if missing (idempotent). Does not run git. */
    init(): Promise<void>
    snapshot(): Promise<CatalogSnapshot>
--  getItem(ref: CatalogRef): Promise<CatalogItemDetail | undefined>
++  /** `known`: the item as listed by a fresh snapshot, reused instead of hashing the item again. */
++  getItem(ref: CatalogRef, known?: CatalogItem): Promise<CatalogItemDetail | undefined>
    /** Absolute paths of all files of an item (folder content for skills, the single file otherwise). */
    itemFiles(ref: CatalogRef): Promise<{ rel: string; abs: string }[]>
    /** Create or update (or rename with `previousName`). Validates names and mcp/hook YAML. */

modified packages/core/src/sync/annotate.ts +23 -10 Viewed

@@ -33,14 +33,17 @@ export async function annotateInventory(
33	33	const snapshot = await catalog.snapshot()
34	34	const catalogItems = new Map(snapshot.items.map((i) => [refKey(i), i]))
35	35	const disk = new DiskReader(inventory.root)
	36	+ const statuses = await Promise.all(
	37	+ inventory.items.map(async (item): Promise<ItemStatus> => {
	38	+ try {
	39	+ return await statusOf(item, inventory.root, lock?.entries ?? [], catalogItems, catalog, disk)
	40	+ } catch {
	41	+ return { managed: false, state: 'not-in-catalog' }
	42	+ }
	43	+ }),
	44	+ )
36	45	const out: Record<string, ItemStatus> = {}
37		- for (const item of inventory.items) {
38		- try {
39		- out[item.id] = await statusOf(item, inventory.root, lock?.entries ?? [], catalogItems, catalog, disk)
40		- } catch {
41		- out[item.id] = { managed: false, state: 'not-in-catalog' }
42		- }
43		- }
	46	+ inventory.items.forEach((item, i) => (out[item.id] = statuses[i]!))
44	47	return out
45	48	}
46	49
@@ -69,10 +72,20 @@ async function statusOf(
69	72	return { managed: false, catalogRef: ref, state: same ? 'in-sync' : 'differs-from-catalog' }
70	73	}
71	74
	75	+/** Comparable form of a path (case-insensitive on Windows), memoized: every item is matched against every lock entry. */
	76	+const pathKeys = new Map<string, string>()
	77	+function pathKey(p: string): string {
	78	+ let k = pathKeys.get(p)
	79	+ if (k === undefined) {
	80	+ k = normalizePath(p)
	81	+ if (process.platform === 'win32') k = k.toLowerCase()
	82	+ pathKeys.set(p, k)
	83	+ }
	84	+ return k
	85	+}
	86	+
72	87	function samePath(root: string, rel: string, abs: string): boolean {
73		- const a = normalizePath(path.resolve(root, rel))
74		- const b = normalizePath(abs)
75		- return process.platform === 'win32' ? a.toLowerCase() === b.toLowerCase() : a === b
	88	+ return pathKey(path.resolve(root, rel)) === pathKey(abs)
76	89	}
77	90
78	91	async function findLockEntry(item: InventoryItem, root: string, entries: LockEntry[], disk: DiskReader): Promise<LockEntry \| undefined> {

    const snapshot = await catalog.snapshot()
    const catalogItems = new Map(snapshot.items.map((i) => [refKey(i), i]))
    const disk = new DiskReader(inventory.root)
++  const statuses = await Promise.all(
++    inventory.items.map(async (item): Promise<ItemStatus> => {
++      try {
++        return await statusOf(item, inventory.root, lock?.entries ?? [], catalogItems, catalog, disk)
++      } catch {
++        return { managed: false, state: 'not-in-catalog' }
++      }
++    }),
++  )
    const out: Record<string, ItemStatus> = {}
--  for (const item of inventory.items) {
++  inventory.items.forEach((item, i) => (out[item.id] = statuses[i]!))
--    try {
--      out[item.id] = await statusOf(item, inventory.root, lock?.entries ?? [], catalogItems, catalog, disk)
--    } catch {
--      out[item.id] = { managed: false, state: 'not-in-catalog' }
--    }
--  }
    return out
  }
    return { managed: false, catalogRef: ref, state: same ? 'in-sync' : 'differs-from-catalog' }
  }
++/** Comparable form of a path (case-insensitive on Windows), memoized: every item is matched against every lock entry. */
++const pathKeys = new Map<string, string>()
++function pathKey(p: string): string {
++  let k = pathKeys.get(p)
++  if (k === undefined) {
++    k = normalizePath(p)
++    if (process.platform === 'win32') k = k.toLowerCase()
++    pathKeys.set(p, k)
++  }
++  return k
++}
++
  function samePath(root: string, rel: string, abs: string): boolean {
--  const a = normalizePath(path.resolve(root, rel))
++  return pathKey(path.resolve(root, rel)) === pathKey(abs)
--  const b = normalizePath(abs)
--  return process.platform === 'win32' ? a.toLowerCase() === b.toLowerCase() : a === b
  }
  async function findLockEntry(item: InventoryItem, root: string, entries: LockEntry[], disk: DiskReader): Promise<LockEntry | undefined> {

modified packages/core/src/sync/disk.ts +2 -4 Viewed

@@ -2,7 +2,7 @@
2	2
3	3	import { readdir, readFile, rm, rmdir, stat } from 'node:fs/promises'
4	4	import path from 'node:path'
5		-import { listFilesRecursive, readText } from '../util.ts'
	5	+import { fileSha256, listFilesRecursive, readText } from '../util.ts'
6	6	import { absPath, sha256, writeFileAtomic } from './helpers.ts'
7	7
8	8	/** Read-only, cached view of a scope. Paths are relative to the root. */
@@ -22,10 +22,8 @@ export class DiskReader {
22	22
23	23	/** sha256 of a regular file, undefined when missing or not a file. */
24	24	async fileHash(rel: string): Promise<string \| undefined> {
25		- const abs = absPath(this.root, rel)
26	25	try {
27		- if (!(await stat(abs)).isFile()) return undefined
28		- return sha256(await readFile(abs))
	26	+ return await fileSha256(absPath(this.root, rel))
29	27	} catch {
30	28	return undefined
31	29	}

  import { readdir, readFile, rm, rmdir, stat } from 'node:fs/promises'
  import path from 'node:path'
--import { listFilesRecursive, readText } from '../util.ts'
++import { fileSha256, listFilesRecursive, readText } from '../util.ts'
  import { absPath, sha256, writeFileAtomic } from './helpers.ts'
  /** Read-only, cached view of a scope. Paths are relative to the root. */
    /** sha256 of a regular file, undefined when missing or not a file. */
    async fileHash(rel: string): Promise<string | undefined> {
--    const abs = absPath(this.root, rel)
      try {
--      if (!(await stat(abs)).isFile()) return undefined
++      return await fileSha256(absPath(this.root, rel))
--      return sha256(await readFile(abs))
      } catch {
        return undefined
      }

modified packages/core/src/sync/plan.ts +33 -25 Viewed

@@ -77,18 +77,25 @@ export async function computePlan(target: SyncTarget, config: ScopeConfig, catal
77	77	}
78	78
79	79	// Desired renders. Items that fail to render keep their lock entry untouched (no action).
	80	+ // Renders and disk comparisons run in parallel (sequential file I/O is slow on Windows); results keep their order.
80	81	const desired: DesiredItem[] = []
81	82	const failed = new Set<string>()
82		- for (const { ref, via } of resolution.refs) {
83		- for (const agent of config.targets) {
84		- try {
85		- const d = await render(ref, agent, via, catalogItems.get(refKey(ref)))
86		- if (d) desired.push(d)
87		- } catch (err) {
88		- failed.add(lockEntryKey({ agent, ...ref }))
89		- warn(`Cannot render ${refKey(ref)} for ${agent}: ${errorMessage(err)}`)
90		- }
91		- }
	83	+ const renders = await Promise.all(
	84	+ resolution.refs.flatMap(({ ref, via }) =>
	85	+ config.targets.map(async (agent): Promise<{ d: DesiredItem \| undefined } \| { ref: CatalogRef; agent: AgentId; err: unknown }> => {
	86	+ try {
	87	+ return { d: await render(ref, agent, via, catalogItems.get(refKey(ref))) }
	88	+ } catch (err) {
	89	+ return { ref, agent, err }
	90	+ }
	91	+ }),
	92	+ ),
	93	+ )
	94	+ for (const r of renders) {
	95	+ if ('err' in r) {
	96	+ failed.add(lockEntryKey({ agent: r.agent, ...r.ref }))
	97	+ warn(`Cannot render ${refKey(r.ref)} for ${r.agent}: ${errorMessage(r.err)}`)
	98	+ } else if (r.d) desired.push(r.d)
92	99	}
93	100	const enabled = new Set(resolution.refs.map((r) => refKey(r.ref)))
94	101	for (const h of await renderHideGlobal({ target, config, layout, enabled, warn })) desired.push({ ...h, via: [HIDE_GLOBAL_VIA] })
@@ -100,25 +107,26 @@ export async function computePlan(target: SyncTarget, config: ScopeConfig, catal
100	107	const lockByKey = new Map((lock?.entries ?? []).map((e) => [lockEntryKey(e), e]))
101	108	const missing = new Set(resolution.missing.map(refKey))
102	109	const actions: InternalAction[] = []
103		- const desiredKeys = new Set<string>()
104		-
105		- for (const d of desired) {
106		- const key = lockEntryKey(d)
107		- desiredKeys.add(key)
	110	+ const desiredKeys = new Set(desired.map(lockEntryKey))
	111	+ const classify = async (
	112	+ item: { type: string; name: string; agent: AgentId },
	113	+ run: () => Promise<InternalAction>,
	114	+ ): Promise<{ action: InternalAction } \| { item: { type: string; name: string; agent: AgentId }; err: unknown }> => {
108	115	try {
109		- actions.push(await classifyDesired(d, lockByKey.get(key), disk))
	116	+ return { action: await run() }
110	117	} catch (err) {
111		- warn(`Cannot plan ${d.type}/${d.name} for ${d.agent}: ${errorMessage(err)}`)
	118	+ return { item, err }
112	119	}
113	120	}
114		- for (const entry of lock?.entries ?? []) {
115		- const key = lockEntryKey(entry)
116		- if (desiredKeys.has(key) \|\| failed.has(key)) continue
117		- try {
118		- actions.push(await classifyStale(entry, missing.has(refKey(entry)), disk))
119		- } catch (err) {
120		- warn(`Cannot plan ${entry.type}/${entry.name} for ${entry.agent}: ${errorMessage(err)}`)
121		- }
	121	+ const classified = await Promise.all([
	122	+ ...desired.map((d) => classify(d, () => classifyDesired(d, lockByKey.get(lockEntryKey(d)), disk))),
	123	+ ...(lock?.entries ?? [])
	124	+ .filter((entry) => !desiredKeys.has(lockEntryKey(entry)) && !failed.has(lockEntryKey(entry)))
	125	+ .map((entry) => classify(entry, () => classifyStale(entry, missing.has(refKey(entry)), disk))),
	126	+ ])
	127	+ for (const r of classified) {
	128	+ if ('action' in r) actions.push(r.action)
	129	+ else warn(`Cannot plan ${r.item.type}/${r.item.name} for ${r.item.agent}: ${errorMessage(r.err)}`)
122	130	}
123	131
124	132	actions.sort((a, b) => compareRefs(a.action, b.action) \|\| a.action.agent.localeCompare(b.action.agent))

    }
    // Desired renders. Items that fail to render keep their lock entry untouched (no action).
++  // Renders and disk comparisons run in parallel (sequential file I/O is slow on Windows); results keep their order.
    const desired: DesiredItem[] = []
    const failed = new Set<string>()
--  for (const { ref, via } of resolution.refs) {
++  const renders = await Promise.all(
--    for (const agent of config.targets) {
++    resolution.refs.flatMap(({ ref, via }) =>
--      try {
++      config.targets.map(async (agent): Promise<{ d: DesiredItem | undefined } | { ref: CatalogRef; agent: AgentId; err: unknown }> => {
--        const d = await render(ref, agent, via, catalogItems.get(refKey(ref)))
++        try {
--        if (d) desired.push(d)
++          return { d: await render(ref, agent, via, catalogItems.get(refKey(ref))) }
--      } catch (err) {
++        } catch (err) {
--        failed.add(lockEntryKey({ agent, ...ref }))
++          return { ref, agent, err }
--        warn(`Cannot render ${refKey(ref)} for ${agent}: ${errorMessage(err)}`)
++        }
--      }
++      }),
--    }
++    ),
++  )
++  for (const r of renders) {
++    if ('err' in r) {
++      failed.add(lockEntryKey({ agent: r.agent, ...r.ref }))
++      warn(`Cannot render ${refKey(r.ref)} for ${r.agent}: ${errorMessage(r.err)}`)
++    } else if (r.d) desired.push(r.d)
    }
    const enabled = new Set(resolution.refs.map((r) => refKey(r.ref)))
    for (const h of await renderHideGlobal({ target, config, layout, enabled, warn })) desired.push({ ...h, via: [HIDE_GLOBAL_VIA] })
    const lockByKey = new Map((lock?.entries ?? []).map((e) => [lockEntryKey(e), e]))
    const missing = new Set(resolution.missing.map(refKey))
    const actions: InternalAction[] = []
--  const desiredKeys = new Set<string>()
++  const desiredKeys = new Set(desired.map(lockEntryKey))
--
++  const classify = async (
--  for (const d of desired) {
++    item: { type: string; name: string; agent: AgentId },
--    const key = lockEntryKey(d)
++    run: () => Promise<InternalAction>,
--    desiredKeys.add(key)
++  ): Promise<{ action: InternalAction } | { item: { type: string; name: string; agent: AgentId }; err: unknown }> => {
      try {
--      actions.push(await classifyDesired(d, lockByKey.get(key), disk))
++      return { action: await run() }
      } catch (err) {
--      warn(`Cannot plan ${d.type}/${d.name} for ${d.agent}: ${errorMessage(err)}`)
++      return { item, err }
      }
    }
--  for (const entry of lock?.entries ?? []) {
++  const classified = await Promise.all([
--    const key = lockEntryKey(entry)
++    ...desired.map((d) => classify(d, () => classifyDesired(d, lockByKey.get(lockEntryKey(d)), disk))),
--    if (desiredKeys.has(key) || failed.has(key)) continue
++    ...(lock?.entries ?? [])
--    try {
++      .filter((entry) => !desiredKeys.has(lockEntryKey(entry)) && !failed.has(lockEntryKey(entry)))
--      actions.push(await classifyStale(entry, missing.has(refKey(entry)), disk))
++      .map((entry) => classify(entry, () => classifyStale(entry, missing.has(refKey(entry)), disk))),
--    } catch (err) {
++  ])
--      warn(`Cannot plan ${entry.type}/${entry.name} for ${entry.agent}: ${errorMessage(err)}`)
++  for (const r of classified) {
--    }
++    if ('action' in r) actions.push(r.action)
++    else warn(`Cannot plan ${r.item.type}/${r.item.name} for ${r.item.agent}: ${errorMessage(r.err)}`)
    }
    actions.sort((a, b) => compareRefs(a.action, b.action) || a.action.agent.localeCompare(b.action.agent))

modified packages/core/src/sync/render/common.ts +3 -3 Viewed

@@ -5,14 +5,14 @@ import { stripBom } from '../../util.ts'
5	5	import { isRecord, joinRel, sha256, type Json } from '../helpers.ts'
6	6	import type { RenderedFile, SourceItem } from './types.ts'
7	7
8		-export function renderedFile(path: string, content: Buffer \| string): RenderedFile {
	8	+export function renderedFile(path: string, content: Buffer \| string, knownSha256?: string): RenderedFile {
9	9	const buf = typeof content === 'string' ? Buffer.from(content, 'utf8') : content
10		- return { path, content: buf, sha256: sha256(buf) }
	10	+ return { path, content: buf, sha256: knownSha256 ?? sha256(buf) }
11	11	}
12	12
13	13	/** Copies every file of a skill folder under `dir`. */
14	14	export function copySkillFiles(source: SourceItem, dir: string): RenderedFile[] {
15		- return source.files.map((f) => renderedFile(joinRel(dir, f.rel), f.content))
	15	+ return source.files.map((f) => renderedFile(joinRel(dir, f.rel), f.content, f.sha256))
16	16	}
17	17
18	18	/** Main file of a single-file item (rule, agent): exact bytes when available. */

  import { isRecord, joinRel, sha256, type Json } from '../helpers.ts'
  import type { RenderedFile, SourceItem } from './types.ts'
--export function renderedFile(path: string, content: Buffer | string): RenderedFile {
++export function renderedFile(path: string, content: Buffer | string, knownSha256?: string): RenderedFile {
    const buf = typeof content === 'string' ? Buffer.from(content, 'utf8') : content
--  return { path, content: buf, sha256: sha256(buf) }
++  return { path, content: buf, sha256: knownSha256 ?? sha256(buf) }
  }
  /** Copies every file of a skill folder under `dir`. */
  export function copySkillFiles(source: SourceItem, dir: string): RenderedFile[] {
--  return source.files.map((f) => renderedFile(joinRel(dir, f.rel), f.content))
++  return source.files.map((f) => renderedFile(joinRel(dir, f.rel), f.content, f.sha256))
  }
  /** Main file of a single-file item (rule, agent): exact bytes when available. */

modified packages/core/src/sync/render/index.ts +6 -2 Viewed

@@ -1,16 +1,20 @@
1	1	import { readFile } from 'node:fs/promises'
2	2	import type { CatalogItem, CatalogRef } from '../../catalog-model.ts'
3	3	import type { CatalogStore } from '../../contracts.ts'
	4	+import { fileSha256 } from '../../util.ts'
4	5	import type { AgentId } from '../../model.ts'
5	6	import { renderClaude } from './claude.ts'
6	7	import { renderCodex } from './codex.ts'
7	8	import type { RenderContext, RenderedItem, SourceItem } from './types.ts'
8	9
9	10	export async function loadSource(catalog: CatalogStore, ref: CatalogRef, item?: CatalogItem): Promise<SourceItem> {
10		- const detail = await catalog.getItem(ref)
	11	+ const detail = await catalog.getItem(ref, item)
11	12	if (!detail) throw new Error(`${ref.type}/${ref.name} not found in the catalog`)
12	13	const files = await Promise.all(
13		- (await catalog.itemFiles(ref)).map(async (f) => ({ rel: f.rel.replace(/\\/g, '/'), content: await readFile(f.abs) })),
	14	+ (await catalog.itemFiles(ref)).map(async (f) => {
	15	+ const [content, sha256] = await Promise.all([readFile(f.abs), fileSha256(f.abs)])
	16	+ return { rel: f.rel.replace(/\\/g, '/'), content, sha256 }
	17	+ }),
14	18	)
15	19	return { item: item ?? detail.item, detail, files }
16	20	}

  import { readFile } from 'node:fs/promises'
  import type { CatalogItem, CatalogRef } from '../../catalog-model.ts'
  import type { CatalogStore } from '../../contracts.ts'
++import { fileSha256 } from '../../util.ts'
  import type { AgentId } from '../../model.ts'
  import { renderClaude } from './claude.ts'
  import { renderCodex } from './codex.ts'
  import type { RenderContext, RenderedItem, SourceItem } from './types.ts'
  export async function loadSource(catalog: CatalogStore, ref: CatalogRef, item?: CatalogItem): Promise<SourceItem> {
--  const detail = await catalog.getItem(ref)
++  const detail = await catalog.getItem(ref, item)
    if (!detail) throw new Error(`${ref.type}/${ref.name} not found in the catalog`)
    const files = await Promise.all(
--    (await catalog.itemFiles(ref)).map(async (f) => ({ rel: f.rel.replace(/\\/g, '/'), content: await readFile(f.abs) })),
++    (await catalog.itemFiles(ref)).map(async (f) => {
++      const [content, sha256] = await Promise.all([readFile(f.abs), fileSha256(f.abs)])
++      return { rel: f.rel.replace(/\\/g, '/'), content, sha256 }
++    }),
    )
    return { item: item ?? detail.item, detail, files }
  }

modified packages/core/src/sync/render/types.ts +2 -1 Viewed

@@ -28,7 +28,8 @@ export interface SourceItem {
28	28	item: CatalogItem
29	29	detail: CatalogItemDetail
30	30	/** All files of the item, relative to the item folder (skills) or the file name. */
31		- files: { rel: string; content: Buffer }[]
	31	+ /** `sha256`: hash of the content when known (cached hash of the catalog file). */
	32	+ files: { rel: string; content: Buffer; sha256?: string }[]
32	33	}
33	34
34	35	export interface RenderContext {

    item: CatalogItem
    detail: CatalogItemDetail
    /** All files of the item, relative to the item folder (skills) or the file name. */
--  files: { rel: string; content: Buffer }[]
++  /** `sha256`: hash of the content when known (cached hash of the catalog file). */
++  files: { rel: string; content: Buffer; sha256?: string }[]
  }
  export interface RenderContext {

modified packages/core/src/util.ts +43 -12 Viewed

@@ -112,23 +112,54 @@ export async function listFilesRecursive(dir: string, limit = 200): Promise<stri
112	112	return out
113	113	}
114	114
115		-/** sha256 of a file, or of all files of a folder (relative path + content, sorted). */
	115	+/*
	116	+ * Hashes are cached for the life of the process, keyed by size and mtime like git's index: skills can weigh tens
	117	+ * of MB and every page hashes the catalog and the scopes, so unchanged files are only stat-ed, never read again.
	118	+ */
	119	+const fileHashes = new Map<string, { size: number; mtimeMs: number; hash: string }>()
	120	+const dirHashes = new Map<string, { signature: string; hash: string }>()
	121	+
	122	+/** sha256 of a regular file's content (cached). Throws when missing or not a file. */
	123	+export async function fileSha256(p: string): Promise<string> {
	124	+ const st = await stat(p)
	125	+ if (!st.isFile()) throw new Error(`Not a file: ${p}`)
	126	+ const cached = fileHashes.get(p)
	127	+ if (cached && cached.size === st.size && cached.mtimeMs === st.mtimeMs) return cached.hash
	128	+ const hash = createHash('sha256').update(await readFile(p)).digest('hex')
	129	+ fileHashes.set(p, { size: st.size, mtimeMs: st.mtimeMs, hash })
	130	+ return hash
	131	+}
	132	+
	133	+/** sha256 of a file, or of all files of a folder (relative path + content, sorted). Cached, see `fileSha256`. */
116	134	export async function hashPath(p: string): Promise<string \| undefined> {
	135	+ return (await pathFingerprint(p)).hash
	136	+}
	137	+
	138	+/** `hashPath` plus the newest mtime of the file or folder content (0 when unknown), in a single walk. */
	139	+export async function pathFingerprint(p: string): Promise<{ hash: string \| undefined; newestMtimeMs: number }> {
117	140	try {
	141	+ if (!(await isDirectory(p))) {
	142	+ const st = await stat(p)
	143	+ return { hash: await fileSha256(p), newestMtimeMs: st.mtimeMs }
	144	+ }
	145	+ const rels = await listFilesRecursive(p, 10_000)
	146	+ const stats = await Promise.all(rels.map((rel) => stat(path.join(p, rel))))
	147	+ const newestMtimeMs = stats.reduce((m, st) => Math.max(m, st.mtimeMs), 0)
	148	+ const signature = rels.map((rel, i) => `${rel}\0${stats[i]!.size}\0${stats[i]!.mtimeMs}`).join('\n')
	149	+ const cached = dirHashes.get(p)
	150	+ if (cached?.signature === signature) return { hash: cached.hash, newestMtimeMs }
118	151	const h = createHash('sha256')
119		- if (await isDirectory(p)) {
120		- for (const rel of await listFilesRecursive(p, 10_000)) {
121		- h.update(rel)
122		- h.update('\0')
123		- h.update(await readFile(path.join(p, rel)))
124		- h.update('\0')
125		- }
126		- } else {
127		- h.update(await readFile(p))
	152	+ for (const rel of rels) {
	153	+ h.update(rel)
	154	+ h.update('\0')
	155	+ h.update(await readFile(path.join(p, rel)))
	156	+ h.update('\0')
128	157	}
129		- return h.digest('hex')
	158	+ const hash = h.digest('hex')
	159	+ dirHashes.set(p, { signature, hash })
	160	+ return { hash, newestMtimeMs }
130	161	} catch {
131		- return undefined
	162	+ return { hash: undefined, newestMtimeMs: 0 }
132	163	}
133	164	}
134	165

    return out
  }
--/** sha256 of a file, or of all files of a folder (relative path + content, sorted). */
++/*
++ * Hashes are cached for the life of the process, keyed by size and mtime like git's index: skills can weigh tens
++ * of MB and every page hashes the catalog and the scopes, so unchanged files are only stat-ed, never read again.
++ */
++const fileHashes = new Map<string, { size: number; mtimeMs: number; hash: string }>()
++const dirHashes = new Map<string, { signature: string; hash: string }>()
++
++/** sha256 of a regular file's content (cached). Throws when missing or not a file. */
++export async function fileSha256(p: string): Promise<string> {
++  const st = await stat(p)
++  if (!st.isFile()) throw new Error(`Not a file: ${p}`)
++  const cached = fileHashes.get(p)
++  if (cached && cached.size === st.size && cached.mtimeMs === st.mtimeMs) return cached.hash
++  const hash = createHash('sha256').update(await readFile(p)).digest('hex')
++  fileHashes.set(p, { size: st.size, mtimeMs: st.mtimeMs, hash })
++  return hash
++}
++
++/** sha256 of a file, or of all files of a folder (relative path + content, sorted). Cached, see `fileSha256`. */
  export async function hashPath(p: string): Promise<string | undefined> {
++  return (await pathFingerprint(p)).hash
++}
++
++/** `hashPath` plus the newest mtime of the file or folder content (0 when unknown), in a single walk. */
++export async function pathFingerprint(p: string): Promise<{ hash: string | undefined; newestMtimeMs: number }> {
    try {
++    if (!(await isDirectory(p))) {
++      const st = await stat(p)
++      return { hash: await fileSha256(p), newestMtimeMs: st.mtimeMs }
++    }
++    const rels = await listFilesRecursive(p, 10_000)
++    const stats = await Promise.all(rels.map((rel) => stat(path.join(p, rel))))
++    const newestMtimeMs = stats.reduce((m, st) => Math.max(m, st.mtimeMs), 0)
++    const signature = rels.map((rel, i) => `${rel}\0${stats[i]!.size}\0${stats[i]!.mtimeMs}`).join('\n')
++    const cached = dirHashes.get(p)
++    if (cached?.signature === signature) return { hash: cached.hash, newestMtimeMs }
      const h = createHash('sha256')
--    if (await isDirectory(p)) {
++    for (const rel of rels) {
--      for (const rel of await listFilesRecursive(p, 10_000)) {
++      h.update(rel)
--        h.update(rel)
++      h.update('\0')
--        h.update('\0')
++      h.update(await readFile(path.join(p, rel)))
--        h.update(await readFile(path.join(p, rel)))
++      h.update('\0')
--        h.update('\0')
--      }
--    } else {
--      h.update(await readFile(p))
      }
--    return h.digest('hex')
++    const hash = h.digest('hex')
++    dirHashes.set(p, { signature, hash })
++    return { hash, newestMtimeMs }
    } catch {
--    return undefined
++    return { hash: undefined, newestMtimeMs: 0 }
    }
  }

modified packages/server/src/context.ts +31 -0 Viewed

@@ -82,6 +82,37 @@ export async function guardGitConflicts(catalog: CatalogStore, git: CatalogGit):
82	82	})
83	83	}
84	84
	85	+/**
	86	+ * Read-only view of the catalog that computes each snapshot, item and file list once. Plan and annotate both read
	87	+ * the whole catalog (hashing every file), and plan loads each item once per target agent: within one read-only
	88	+ * request the catalog does not change, so the work is shared. Never use it across a write.
	89	+ */
	90	+export function memoizeCatalogReads(catalog: CatalogStore): CatalogStore {
	91	+ let snapshot: ReturnType<CatalogStore['snapshot']> \| undefined
	92	+ const items = new Map<string, ReturnType<CatalogStore['getItem']>>()
	93	+ const files = new Map<string, ReturnType<CatalogStore['itemFiles']>>()
	94	+ const memo = <T>(cache: Map<string, T>, key: string, load: () => T): T => {
	95	+ let v = cache.get(key)
	96	+ if (v === undefined) {
	97	+ v = load()
	98	+ cache.set(key, v)
	99	+ }
	100	+ return v
	101	+ }
	102	+ const overrides: Partial<CatalogStore> = {
	103	+ snapshot: () => (snapshot ??= catalog.snapshot()),
	104	+ getItem: (ref, known) => memo(items, `${ref.type}/${ref.name}`, () => catalog.getItem(ref, known)),
	105	+ itemFiles: (ref) => memo(files, `${ref.type}/${ref.name}`, () => catalog.itemFiles(ref)),
	106	+ }
	107	+ return new Proxy(catalog, {
	108	+ get(target, prop) {
	109	+ if (prop in overrides) return overrides[prop as keyof CatalogStore]
	110	+ const v: unknown = Reflect.get(target, prop)
	111	+ return typeof v === 'function' ? v.bind(target) : v
	112	+ },
	113	+ })
	114	+}
	115	+
85	116	export async function isDirectory(p: string): Promise<boolean> {
86	117	try {
87	118	return (await stat(p)).isDirectory()

    })
  }
++/**
++ * Read-only view of the catalog that computes each snapshot, item and file list once. Plan and annotate both read
++ * the whole catalog (hashing every file), and plan loads each item once per target agent: within one read-only
++ * request the catalog does not change, so the work is shared. Never use it across a write.
++ */
++export function memoizeCatalogReads(catalog: CatalogStore): CatalogStore {
++  let snapshot: ReturnType<CatalogStore['snapshot']> | undefined
++  const items = new Map<string, ReturnType<CatalogStore['getItem']>>()
++  const files = new Map<string, ReturnType<CatalogStore['itemFiles']>>()
++  const memo = <T>(cache: Map<string, T>, key: string, load: () => T): T => {
++    let v = cache.get(key)
++    if (v === undefined) {
++      v = load()
++      cache.set(key, v)
++    }
++    return v
++  }
++  const overrides: Partial<CatalogStore> = {
++    snapshot: () => (snapshot ??= catalog.snapshot()),
++    getItem: (ref, known) => memo(items, `${ref.type}/${ref.name}`, () => catalog.getItem(ref, known)),
++    itemFiles: (ref) => memo(files, `${ref.type}/${ref.name}`, () => catalog.itemFiles(ref)),
++  }
++  return new Proxy(catalog, {
++    get(target, prop) {
++      if (prop in overrides) return overrides[prop as keyof CatalogStore]
++      const v: unknown = Reflect.get(target, prop)
++      return typeof v === 'function' ? v.bind(target) : v
++    },
++  })
++}
++
  export async function isDirectory(p: string): Promise<boolean> {
    try {
      return (await stat(p)).isDirectory()

modified packages/server/src/scope-routes.ts +18 -19 Viewed

@@ -17,7 +17,7 @@ import {
17	17	type SyncPlan,
18	18	} from '@skilly/core'
19	19	import type { Hono } from 'hono'
20		-import { HttpError, isStringArray, readJsonBody, type AppContext, type ScopeRef } from './context.ts'
	20	+import { HttpError, isStringArray, memoizeCatalogReads, readJsonBody, type AppContext, type ScopeRef } from './context.ts'
21	21
22	22	const RESOLUTIONS: readonly string[] = ['keep-local', 'take-catalog', 'push-to-catalog']
23	23	const HOME_TYPES = new Set(['skill', 'rule', 'agent'])
@@ -82,26 +82,25 @@ export function registerScopeRoutes(app: Hono, ctx: AppContext): void {
82	82	const { syncEngine: engine } = ctx
83	83
84	84	async function buildState(scope: ScopeRef): Promise<ScopeState> {
85		- const catalog = await ctx.catalog()
86		- const catalogExists = await catalog.exists()
87		- const inventory = await scope.scan()
88		- const config = (await engine.readConfig(scope.target)) ?? null
89		- const lock = (await engine.readLock(scope.target)) ?? null
90		- let plan: SyncPlan \| null = null
91		- if (config) {
92		- try {
93		- plan = await engine.plan(scope.target, config, catalog)
94		- } catch (e) {
95		- // A missing catalog folder is not an error for the UI: there is just nothing to plan against yet.
96		- if (catalogExists) throw e
97		- }
98		- }
99		- let statuses: ScopeState['statuses'] = {}
100		- try {
101		- statuses = await engine.annotate(inventory, catalog, lock ?? undefined)
102		- } catch (e) {
	85	+ const catalog = memoizeCatalogReads(await ctx.catalog())
	86	+ const [catalogExists, config, lock] = await Promise.all([
	87	+ catalog.exists(),
	88	+ engine.readConfig(scope.target).then((c) => c ?? null),
	89	+ engine.readLock(scope.target).then((l) => l ?? null),
	90	+ ])
	91	+ // A missing catalog folder is not an error for the UI: there is just nothing to plan against yet.
	92	+ const tolerate = <T>(fallback: T) => (e: unknown): T => {
103	93	if (catalogExists) throw e
	94	+ return fallback
104	95	}
	96	+ // Plan and scan + annotate are independent: they run side by side and share the catalog reads.
	97	+ const [plan, [inventory, statuses]] = await Promise.all([
	98	+ config ? engine.plan(scope.target, config, catalog).catch(tolerate<SyncPlan \| null>(null)) : null,
	99	+ scope.scan().then(async (inv) => {
	100	+ const st = await engine.annotate(inv, catalog, lock ?? undefined).catch(tolerate<ScopeState['statuses']>({}))
	101	+ return [inv, st] as const
	102	+ }),
	103	+ ])
105	104	return { scope: scope.id, config, inventory, statuses, plan, lock }
106	105	}
107	106

    type SyncPlan,
  } from '@skilly/core'
  import type { Hono } from 'hono'
--import { HttpError, isStringArray, readJsonBody, type AppContext, type ScopeRef } from './context.ts'
++import { HttpError, isStringArray, memoizeCatalogReads, readJsonBody, type AppContext, type ScopeRef } from './context.ts'
  const RESOLUTIONS: readonly string[] = ['keep-local', 'take-catalog', 'push-to-catalog']
  const HOME_TYPES = new Set(['skill', 'rule', 'agent'])
    const { syncEngine: engine } = ctx
    async function buildState(scope: ScopeRef): Promise<ScopeState> {
--    const catalog = await ctx.catalog()
++    const catalog = memoizeCatalogReads(await ctx.catalog())
--    const catalogExists = await catalog.exists()
++    const [catalogExists, config, lock] = await Promise.all([
--    const inventory = await scope.scan()
++      catalog.exists(),
--    const config = (await engine.readConfig(scope.target)) ?? null
++      engine.readConfig(scope.target).then((c) => c ?? null),
--    const lock = (await engine.readLock(scope.target)) ?? null
++      engine.readLock(scope.target).then((l) => l ?? null),
--    let plan: SyncPlan | null = null
++    ])
--    if (config) {
++    // A missing catalog folder is not an error for the UI: there is just nothing to plan against yet.
--      try {
++    const tolerate = <T>(fallback: T) => (e: unknown): T => {
--        plan = await engine.plan(scope.target, config, catalog)
--      } catch (e) {
--        // A missing catalog folder is not an error for the UI: there is just nothing to plan against yet.
--        if (catalogExists) throw e
--      }
--    }
--    let statuses: ScopeState['statuses'] = {}
--    try {
--      statuses = await engine.annotate(inventory, catalog, lock ?? undefined)
--    } catch (e) {
        if (catalogExists) throw e
++      return fallback
      }
++    // Plan and scan + annotate are independent: they run side by side and share the catalog reads.
++    const [plan, [inventory, statuses]] = await Promise.all([
++      config ? engine.plan(scope.target, config, catalog).catch(tolerate<SyncPlan | null>(null)) : null,
++      scope.scan().then(async (inv) => {
++        const st = await engine.annotate(inv, catalog, lock ?? undefined).catch(tolerate<ScopeState['statuses']>({}))
++        return [inv, st] as const
++      }),
++    ])
      return { scope: scope.id, config, inventory, statuses, plan, lock }
    }

perf: cache file hashes and share catalog reads to speed up scope pages