Ayuriel/openclaw
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: restore memory search output dimensionality

Browse Source
This commit is contained in:
Marc J Saint-jour 2026-03-12 20:04:18 -04:00
parent 893cbe0cf2
commit 1de38237e7
1 changed files with 11 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

167
src/memory/manager-sync-ops.ts
View File

@ -29,18 +29,12 @@ import { isFileMissingError } from "./fs-utils.js";
import {
buildFileEntry,
ensureDir,
hashText,
listMemoryFiles,
normalizeExtraMemoryPaths,
runWithConcurrency,
} from "./internal.js";
import { type MemoryFileEntry } from "./internal.js";
import { ensureMemoryIndexSchema } from "./memory-schema.js";
import {
buildCaseInsensitiveExtensionGlob,
classifyMemoryMultimodalPath,
getMemoryMultimodalExtensions,
} from "./multimodal.js";
import type { SessionFileEntry } from "./session-files.js";
import {
buildSessionEntry,
@ -56,7 +50,6 @@ type MemoryIndexMeta = {
provider: string;
providerKey?: string;
sources?: MemorySource[];
scopeHash?: string;
chunkTokens: number;
chunkOverlap: number;
vectorDims?: number;
@ -151,8 +144,6 @@ export abstract class MemoryManagerSyncOps {
protected abstract sync(params?: {
reason?: string;
force?: boolean;
forceSessions?: boolean;
sessionFile?: string;
progress?: (update: MemorySyncProgressUpdate) => void;
}): Promise<void>;
protected abstract withTimeout<T>(
@ -392,22 +383,9 @@ export abstract class MemoryManagerSyncOps {
}
if (stat.isDirectory()) {
watchPaths.add(path.join(entry, "**", "*.md"));
if (this.settings.multimodal.enabled) {
for (const modality of this.settings.multimodal.modalities) {
for (const extension of getMemoryMultimodalExtensions(modality)) {
watchPaths.add(
path.join(entry, "**", buildCaseInsensitiveExtensionGlob(extension)),
);
}
}
}
continue;
}
if (
stat.isFile() &&
(entry.toLowerCase().endsWith(".md") ||
classifyMemoryMultimodalPath(entry, this.settings.multimodal) !== null)
) {
if (stat.isFile() && entry.toLowerCase().endsWith(".md")) {
watchPaths.add(entry);
}
} catch {
@ -613,35 +591,6 @@ export abstract class MemoryManagerSyncOps {
return resolvedFile.startsWith(`${resolvedDir}${path.sep}`);
}
private normalizeTargetSessionFiles(sessionFiles?: string[]): Set<string> | null {
if (!sessionFiles || sessionFiles.length === 0) {
return null;
}
const normalized = new Set<string>();
for (const sessionFile of sessionFiles) {
const trimmed = sessionFile.trim();
if (!trimmed) {
continue;
}
const resolved = path.resolve(trimmed);
if (this.isSessionFileForAgent(resolved)) {
normalized.add(resolved);
}
}
return normalized.size > 0 ? normalized : null;
}
private clearSyncedSessionFiles(targetSessionFiles?: Iterable<string> | null) {
if (!targetSessionFiles) {
this.sessionsDirtyFiles.clear();
} else {
for (const targetSessionFile of targetSessionFiles) {
this.sessionsDirtyFiles.delete(targetSessionFile);
}
}
this.sessionsDirty = this.sessionsDirtyFiles.size > 0;
}
protected ensureIntervalSync() {
const minutes = this.settings.sync.intervalMinutes;
if (!minutes || minutes <= 0 || this.intervalTimer) {
@ -671,15 +620,12 @@ export abstract class MemoryManagerSyncOps {
}
private shouldSyncSessions(
params?: { reason?: string; force?: boolean; sessionFiles?: string[] },
params?: { reason?: string; force?: boolean },
needsFullReindex = false,
) {
if (!this.sources.has("sessions")) {
return false;
}
if (params?.sessionFiles?.some((sessionFile) => sessionFile.trim().length > 0)) {
return true;
}
if (params?.force) {
return true;
}
@ -703,19 +649,9 @@ export abstract class MemoryManagerSyncOps {
return;
}
const files = await listMemoryFiles(
this.workspaceDir,
this.settings.extraPaths,
this.settings.multimodal,
);
const files = await listMemoryFiles(this.workspaceDir, this.settings.extraPaths);
const fileEntries = (
await runWithConcurrency(
files.map(
(file) => async () =>
await buildFileEntry(file, this.workspaceDir, this.settings.multimodal),
),
this.getIndexConcurrency(),
)
await Promise.all(files.map(async (file) => buildFileEntry(file, this.workspaceDir)))
).filter((entry): entry is MemoryFileEntry => entry !== null);
log.debug("memory sync: indexing memory files", {
files: fileEntries.length,
@ -786,7 +722,6 @@ export abstract class MemoryManagerSyncOps {
private async syncSessionFiles(params: {
needsFullReindex: boolean;
targetSessionFiles?: string[];
progress?: MemorySyncProgressState;
}) {
// FTS-only mode: skip embedding sync (no provider)
@ -795,22 +730,13 @@ export abstract class MemoryManagerSyncOps {
return;
}
const targetSessionFiles = params.needsFullReindex
? null
: this.normalizeTargetSessionFiles(params.targetSessionFiles);
const files = targetSessionFiles
? Array.from(targetSessionFiles)
: await listSessionFilesForAgent(this.agentId);
const activePaths = targetSessionFiles
? null
: new Set(files.map((file) => sessionPathForFile(file)));
const indexAll =
params.needsFullReindex || Boolean(targetSessionFiles) || this.sessionsDirtyFiles.size === 0;
const files = await listSessionFilesForAgent(this.agentId);
const activePaths = new Set(files.map((file) => sessionPathForFile(file)));
const indexAll = params.needsFullReindex || this.sessionsDirtyFiles.size === 0;
log.debug("memory sync: indexing session files", {
files: files.length,
indexAll,
dirtyFiles: this.sessionsDirtyFiles.size,
targetedFiles: targetSessionFiles?.size ?? 0,
batch: this.batch.enabled,
concurrency: this.getIndexConcurrency(),
});
@ -871,12 +797,6 @@ export abstract class MemoryManagerSyncOps {
});
await runWithConcurrency(tasks, this.getIndexConcurrency());
if (activePaths === null) {
// Targeted syncs only refresh the requested transcripts and should not
// prune unrelated session rows without a full directory enumeration.
return;
}
const staleRows = this.db
.prepare(`SELECT path FROM files WHERE source = ?`)
.all("sessions") as Array<{ path: string }>;
@ -935,7 +855,6 @@ export abstract class MemoryManagerSyncOps {
protected async runSync(params?: {
reason?: string;
force?: boolean;
sessionFiles?: string[];
progress?: (update: MemorySyncProgressUpdate) => void;
}) {
const progress = params?.progress ? this.createSyncProgress(params.progress) : undefined;
@ -949,54 +868,13 @@ export abstract class MemoryManagerSyncOps {
const vectorReady = await this.ensureVectorReady();
const meta = this.readMeta();
const configuredSources = this.resolveConfiguredSourcesForMeta();
const configuredScopeHash = this.resolveConfiguredScopeHash();
const targetSessionFiles = this.normalizeTargetSessionFiles(params?.sessionFiles);
const hasTargetSessionFiles = targetSessionFiles !== null;
if (hasTargetSessionFiles && targetSessionFiles && this.sources.has("sessions")) {
// Post-compaction refreshes should only update the explicit transcript files and
// leave broader reindex/dirty-work decisions to the regular sync path.
try {
await this.syncSessionFiles({
needsFullReindex: false,
targetSessionFiles: Array.from(targetSessionFiles),
progress: progress ?? undefined,
});
this.clearSyncedSessionFiles(targetSessionFiles);
} catch (err) {
const reason = err instanceof Error ? err.message : String(err);
const activated =
this.shouldFallbackOnError(reason) && (await this.activateFallbackProvider(reason));
if (activated) {
if (
process.env.OPENCLAW_TEST_FAST === "1" &&
process.env.OPENCLAW_TEST_MEMORY_UNSAFE_REINDEX === "1"
) {
await this.runUnsafeReindex({
reason: params?.reason,
force: true,
progress: progress ?? undefined,
});
} else {
await this.runSafeReindex({
reason: params?.reason,
force: true,
progress: progress ?? undefined,
});
}
return;
}
throw err;
}
return;
}
const needsFullReindex =
(params?.force && !hasTargetSessionFiles) ||
params?.force ||
!meta ||
(this.provider && meta.model !== this.provider.model) ||
(this.provider && meta.provider !== this.provider.id) ||
meta.providerKey !== this.providerKey ||
this.metaSourcesDiffer(meta, configuredSources) ||
meta.scopeHash !== configuredScopeHash ||
meta.chunkTokens !== this.settings.chunking.tokens ||
meta.chunkOverlap !== this.settings.chunking.overlap ||
(vectorReady && !meta?.vectorDims);
@ -1022,8 +900,7 @@ export abstract class MemoryManagerSyncOps {
}
const shouldSyncMemory =
this.sources.has("memory") &&
((!hasTargetSessionFiles && params?.force) || needsFullReindex || this.dirty);
this.sources.has("memory") && (params?.force || needsFullReindex || this.dirty);
const shouldSyncSessions = this.shouldSyncSessions(params, needsFullReindex);
if (shouldSyncMemory) {
@ -1032,11 +909,7 @@ export abstract class MemoryManagerSyncOps {
}
if (shouldSyncSessions) {
await this.syncSessionFiles({
needsFullReindex,
targetSessionFiles: targetSessionFiles ? Array.from(targetSessionFiles) : undefined,
progress: progress ?? undefined,
});
await this.syncSessionFiles({ needsFullReindex, progress: progress ?? undefined });
this.sessionsDirty = false;
this.sessionsDirtyFiles.clear();
} else if (this.sessionsDirtyFiles.size > 0) {
@ -1121,9 +994,9 @@ export abstract class MemoryManagerSyncOps {
config: this.cfg,
agentDir: resolveAgentDir(this.cfg, this.agentId),
provider: fallback,
outputDimensionality: this.settings.outputDimensionality,
remote: this.settings.remote,
model: fallbackModel,
outputDimensionality: this.settings.outputDimensionality,
fallback: "none",
local: this.settings.local,
});
@ -1215,7 +1088,6 @@ export abstract class MemoryManagerSyncOps {
provider: this.provider?.id ?? "none",
providerKey: this.providerKey!,
sources: this.resolveConfiguredSourcesForMeta(),
scopeHash: this.resolveConfiguredScopeHash(),
chunkTokens: this.settings.chunking.tokens,
chunkOverlap: this.settings.chunking.overlap,
};
@ -1287,7 +1159,6 @@ export abstract class MemoryManagerSyncOps {
provider: this.provider?.id ?? "none",
providerKey: this.providerKey!,
sources: this.resolveConfiguredSourcesForMeta(),
scopeHash: this.resolveConfiguredScopeHash(),
chunkTokens: this.settings.chunking.tokens,
chunkOverlap: this.settings.chunking.overlap,
};
@ -1365,22 +1236,6 @@ export abstract class MemoryManagerSyncOps {
return normalized.length > 0 ? normalized : ["memory"];
}
private resolveConfiguredScopeHash(): string {
const extraPaths = normalizeExtraMemoryPaths(this.workspaceDir, this.settings.extraPaths)
.map((value) => value.replace(/\\/g, "/"))
.toSorted();
return hashText(
JSON.stringify({
extraPaths,
multimodal: {
enabled: this.settings.multimodal.enabled,
modalities: [...this.settings.multimodal.modalities].toSorted(),
maxFileBytes: this.settings.multimodal.maxFileBytes,
},
}),
);
}
private metaSourcesDiffer(meta: MemoryIndexMeta, configuredSources: MemorySource[]): boolean {
const metaSources = this.normalizeMetaSources(meta);
if (metaSources.length !== configuredSources.length) {