Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
7d2207f
feat(ai): define retrieval api contract and configuration
aotenjou Sep 7, 2026
c928d66
fix(ai): harden retrieval api validation
aotenjou Sep 10, 2026
9818d52
feat(ai): add versioned ingestion metadata and import state model
aotenjou Sep 11, 2026
79a8451
test(ai): add deterministic retrieval fixture and HTTP contract baseline
aotenjou Sep 11, 2026
07bbd25
chore(ai): preserve week1 retrieval baseline
aotenjou Sep 26, 2026
abbad8b
feat(graphrag): add verified dataset source loader
aotenjou Sep 26, 2026
ceb101d
chore(graphrag): remove generated Python caches
aotenjou Sep 26, 2026
8ebfc8b
fix(graphrag): harden source verification and streaming
aotenjou Sep 26, 2026
a4350af
chore(graphrag): remove generated Python caches
aotenjou Sep 26, 2026
87f1889
fix(graphrag): verify staged downloads and strict JSON arrays
aotenjou Sep 26, 2026
3400599
feat(graphrag): add deterministic document normalization and chunking
aotenjou Sep 26, 2026
bee429d
fix(graphrag): preserve source identity and Java-compatible offsets
aotenjou Sep 26, 2026
b825c9f
feat(graphrag): build deterministic entity graph artifacts
aotenjou Sep 26, 2026
a125601
fix(graphrag): preserve dataset evidence triples
aotenjou Sep 26, 2026
c4157df
fix(graphrag): make graph publication deterministic and staged
aotenjou Sep 26, 2026
985fba0
feat(ai): add deterministic Lucene BM25 index builder
aotenjou Sep 26, 2026
deea988
feat(ai): add offline versioned vector index builder
aotenjou Sep 26, 2026
9a0d2c3
feat(graphrag): complete week2 ingestion and readiness pipeline
aotenjou Sep 26, 2026
ace2021
fix(graphrag): harden ingestion and index artifact validation
aotenjou Sep 26, 2026
e2f59fe
test(ai): align index fixtures with chunking contract
aotenjou Sep 26, 2026
c9e9d02
feat(graphrag): publish validated retrieval artifacts
aotenjou Oct 8, 2026
d9394f8
feat(ai): add budgeted hybrid recall with graph channel
aotenjou Oct 8, 2026
0e3b79d
feat(ai): extend retrieval contract for hybrid recall
aotenjou Oct 8, 2026
98cd90f
docs(ai): document fixed hybrid recall MVP
aotenjou Oct 8, 2026
3676d43
fix(ai): make ingestion and artifact publication exclusive
aotenjou Oct 10, 2026
4218914
fix(ai): secure and cache memory retrieval service
aotenjou Oct 10, 2026
1dc145a
fix(ai): preserve retrieval provenance and trace identity
aotenjou Oct 10, 2026
a93c597
fix(graphrag): make dataset ingestion identities and spills determini…
aotenjou Oct 10, 2026
e025eb3
docs(ai): update MVP review verification
aotenjou Oct 10, 2026
a8c3ce8
docs(ai): remove obsolete retrieval README files
aotenjou Oct 12, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
245 changes: 225 additions & 20 deletions geaflow-ai/src/main/java/org/apache/geaflow/ai/GeaFlowMemoryServer.java

Large diffs are not rendered by default.

186 changes: 183 additions & 3 deletions geaflow-ai/src/main/java/org/apache/geaflow/ai/GraphMemoryServer.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,20 @@
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
import org.apache.geaflow.ai.graph.GraphAccessor;
import org.apache.geaflow.ai.graph.GraphEntity;
import org.apache.geaflow.ai.graph.GraphVertex;
import org.apache.geaflow.ai.index.EmbeddingIndexStore;
import org.apache.geaflow.ai.index.EntityAttributeIndexStore;
import org.apache.geaflow.ai.index.IndexStore;
import org.apache.geaflow.ai.operator.EmbeddingOperator;
import org.apache.geaflow.ai.operator.GraphSearchStore;
import org.apache.geaflow.ai.operator.GraphSearchStore.ScoredGraphEntity;
import org.apache.geaflow.ai.operator.SearchOperator;
import org.apache.geaflow.ai.operator.SessionOperator;
import org.apache.geaflow.ai.search.VectorSearch;
Expand All @@ -38,15 +45,27 @@
import org.apache.geaflow.ai.verbalization.Context;
import org.apache.geaflow.ai.verbalization.VerbalizationFunction;

public class GraphMemoryServer {
public class GraphMemoryServer implements AutoCloseable {

private final SessionManagement sessionManagement = new SessionManagement();
private final List<GraphAccessor> graphAccessors = new ArrayList<>();
private final List<IndexStore> indexStores = new ArrayList<>();
private final ReadWriteLock keywordStoreLock = new ReentrantReadWriteLock();
private GraphSearchStore keywordSearchStore;
private GraphAccessor keywordSearchAccessor;
private EntityAttributeIndexStore keywordSearchIndex;
private int keywordSearchStoreBuildCount;
private boolean closed;

public void addGraphAccessor(GraphAccessor graph) {
if (graph != null) {
graphAccessors.add(graph);
keywordStoreLock.writeLock().lock();
try {
closeKeywordSearchStore();
graphAccessors.add(graph);
} finally {
keywordStoreLock.writeLock().unlock();
}
}
}

Expand All @@ -56,14 +75,175 @@ public List<GraphAccessor> getGraphAccessors() {

public void addIndexStore(IndexStore indexStore) {
if (indexStore != null) {
indexStores.add(indexStore);
keywordStoreLock.writeLock().lock();
try {
closeKeywordSearchStore();
indexStores.add(indexStore);
} finally {
keywordStoreLock.writeLock().unlock();
}
}
}

public List<IndexStore> getIndexStores() {
return indexStores;
}

/** Executes bounded keyword retrieval without creating a session. */
public List<ScoredGraphEntity> searchKeyword(String query, int topK, int maxCandidates,
long deadlineNanos) {
if (graphAccessors.isEmpty()) {
throw new IllegalStateException("No graph accessor available");
}
EntityAttributeIndexStore keywordIndex = findKeywordIndex();
if (keywordIndex == null) {
throw new IllegalStateException("Keyword index is not available");
}
GraphAccessor accessor = graphAccessors.get(0);
GraphSearchStore store = acquireKeywordSearchStore(accessor, keywordIndex, deadlineNanos);
try {
if (System.nanoTime() > deadlineNanos) {
throw retrievalTimeout();
}
return store.searchScored(query, accessor, topK, maxCandidates, deadlineNanos);
} finally {
keywordStoreLock.readLock().unlock();
}
}

/** Invalidates the cached keyword index after graph data or schemas change. */
public void invalidateKeywordSearchIndex() {
keywordStoreLock.writeLock().lock();
try {
closeKeywordSearchStore();
} finally {
keywordStoreLock.writeLock().unlock();
}
}

@Override
public void close() {
keywordStoreLock.writeLock().lock();
try {
closed = true;
closeKeywordSearchStore();
} finally {
keywordStoreLock.writeLock().unlock();
}
}

public int getKeywordSearchStoreBuildCount() {
keywordStoreLock.readLock().lock();
try {
return keywordSearchStoreBuildCount;
} finally {
keywordStoreLock.readLock().unlock();
}
}

private GraphSearchStore acquireKeywordSearchStore(GraphAccessor accessor,
EntityAttributeIndexStore keywordIndex,
long deadlineNanos) {
lockUntilDeadline(keywordStoreLock.readLock(), deadlineNanos);
if (closed) {
keywordStoreLock.readLock().unlock();
throw new IllegalStateException("graph memory server is closed");
}
if (keywordSearchStore != null && keywordSearchAccessor == accessor
&& keywordSearchIndex == keywordIndex) {
return keywordSearchStore;
}
keywordStoreLock.readLock().unlock();
lockUntilDeadline(keywordStoreLock.writeLock(), deadlineNanos);
try {
if (closed) {
throw new IllegalStateException("graph memory server is closed");
}
if (keywordSearchStore == null || keywordSearchAccessor != accessor
|| keywordSearchIndex != keywordIndex) {
closeKeywordSearchStore();
keywordSearchStore = buildKeywordSearchStore(accessor, keywordIndex, deadlineNanos);
keywordSearchAccessor = accessor;
keywordSearchIndex = keywordIndex;
keywordSearchStoreBuildCount++;
}
keywordStoreLock.readLock().lock();
return keywordSearchStore;
} finally {
keywordStoreLock.writeLock().unlock();
}
}

private static void lockUntilDeadline(Lock lock, long deadlineNanos) {
long remaining = deadlineNanos - System.nanoTime();
try {
if (remaining <= 0 || !lock.tryLock(remaining, TimeUnit.NANOSECONDS)) {
throw retrievalTimeout();
}
} catch (InterruptedException error) {
Thread.currentThread().interrupt();
throw retrievalTimeout();
}
}

private static GraphSearchStore buildKeywordSearchStore(GraphAccessor accessor,
EntityAttributeIndexStore keywordIndex,
long deadlineNanos) {
if (!keywordIndex.isInitialized()) {
throw new IllegalStateException("Keyword index is not initialized");
}
GraphSearchStore store = new GraphSearchStore();
boolean complete = false;
try {
java.util.Iterator<GraphVertex> vertices = accessor.scanVertex();
while (vertices.hasNext()) {
if (System.nanoTime() > deadlineNanos) {
throw retrievalTimeout();
}
GraphVertex vertex = vertices.next();
List<org.apache.geaflow.ai.index.vector.IVector> vectors = keywordIndex.getEntityIndex(vertex);
if (vectors != null && !vectors.isEmpty()) {
store.indexVertex(vertex, vectors);
}
}
if (System.nanoTime() > deadlineNanos) {
throw retrievalTimeout();
}
store.finishWriting();
complete = true;
return store;
} finally {
if (!complete) {
store.close();
}
}
Comment on lines +92 to +219
}

private EntityAttributeIndexStore findKeywordIndex() {
for (IndexStore indexStore : indexStores) {
if (indexStore instanceof EntityAttributeIndexStore) {
return (EntityAttributeIndexStore) indexStore;
}
}
return null;
}

private void closeKeywordSearchStore() {
if (keywordSearchStore != null) {
final GraphSearchStore previous = keywordSearchStore;
keywordSearchStore = null;
keywordSearchAccessor = null;
keywordSearchIndex = null;
previous.close();
}
}

private static org.apache.geaflow.ai.retrieval.api.model.RetrievalException retrievalTimeout() {
return new org.apache.geaflow.ai.retrieval.api.model.RetrievalException(
org.apache.geaflow.ai.retrieval.api.model.RetrievalErrorCode.RETRIEVAL_TIMEOUT,
"retrieval deadline exceeded");
}

public String createSession() {
String sessionId = sessionManagement.createSession();
if (sessionId == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,11 @@

public class EntityAttributeIndexStore implements IndexStore {

private VerbalizationFunction verbFunc;
private volatile VerbalizationFunction verbFunc;

public boolean isInitialized() {
return verbFunc != null;
}

public void initStore(VerbalizationFunction func) {
if (func != null) {
Expand All @@ -41,6 +45,9 @@ public void initStore(VerbalizationFunction func) {

@Override
public List<IVector> getEntityIndex(GraphEntity entity) {
if (!isInitialized()) {
throw new IllegalStateException("Entity attribute index is not initialized");
}
if (entity instanceof GraphVertex) {
String verbalization = verbFunc.verbalize(new SubGraph().addVertex((GraphVertex) entity));
List<String> sentences = new ArrayList<>();
Expand Down
Loading
Loading