Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions native/src/main/java/org/humemai/nativelib/NativeExtras.java
Original file line number Diff line number Diff line change
Expand Up @@ -315,8 +315,9 @@ private static String q(final String text) {
}

/**
* ops: metadata {type, prop} | approximate {type, prop, vector, k, allowed}. The index is the one the type has on the property; its bucket
* indexes are searched in turn (the caller merges). metadata describes the first bucket index, as the JPype wrapper does.
* ops: metadata {type, prop} | approximate {type, prop, vector, k, allowed} | build {type, prop}. The index is the one the type has on the
* property; its bucket indexes are searched in turn (the caller merges). metadata describes the first bucket index, as the JPype wrapper does.
* build rebuilds the graph of every bucket index now (buildVectorGraphNow), as the JPype wrapper's build_graph_now() does (#162).
*/
@CEntryPoint(name = "arcadedb_n_vindex")
@SuppressWarnings("unchecked")
Expand Down Expand Up @@ -383,6 +384,10 @@ public static CCharPointer vindex(final IsolateThread thread, final long dbh, fi
}
sb.append(']');
}
case "build" -> {
for (final LSMVectorIndex lsm : parts)
lsm.buildVectorGraphNow();
}
default -> throw new IllegalArgumentException("unknown vector index op " + ArcadeNative.str(op));
}
return ArcadeNative.out(sb.append('}').toString());
Expand Down
20 changes: 15 additions & 5 deletions src/arcadedb/_native/schema_objects.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,17 +136,27 @@ def countEntries(self):
return int(row.get("c"))

def warmUp(self):
# the engine loads a persisted graph on the first search: run that search now
self.buildVectorGraphNow()

def buildVectorGraphNow(self):
# the engine builds the graph on the first search: one search now, on a vector of the index's dimension
# the engine loads a persisted graph on the first search: one search now, on a vector of the index's dimension
dims = self.engine_info()["dimensions"]
if dims:
self._db.query(
"sql", "SELECT vectorNeighbors(?, ?, 1)", self._name, [1.0] * dims
).close()

def buildVectorGraphNow(self):
# the engine's own rebuild of every bucket index (LSMVectorIndex.buildVectorGraphNow), as the JPype wrapper calls it:
# a search builds a graph only when there is none, so it left the vectors written since in the delta (#162)
from . import lib
from .extras import _call, _enc

_call(
"Failed to rebuild vector graph",
"arcadedb_n_vindex",
self._db._h,
lib.b("build"),
lib.b(_enc({"type": self._type, "prop": self._prop})),
)


def _rid_text(value) -> str:
get_rid = getattr(value, "get_rid", None)
Expand Down
38 changes: 38 additions & 0 deletions tests/test_vector_index_introspection.py
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,44 @@ def test_alter_database_sets_the_ratio_the_threshold_reads(self, temp_db):
assert index.get_stats()["effectiveMutationsThreshold"] == 500


class TestBuildGraphNow:
"""``build_graph_now()`` rebuilds a graph that already exists, so the vectors written since it was built join it (#162).

The native build ran one search, which builds a graph only when there is none: on a built index the call returned at
once and the new vectors stayed in the delta. The automatic triggers are off (a high floor, no inactivity timer), so
only the call can move the counters. The graph starts at 1000 vectors because the engine rebuilds a smaller one
synchronously on the next search (ASYNC_REBUILD_MIN_GRAPH_SIZE), which hid the defect.
"""

ROWS = [
(f"v{i}", [1.0 + i % 10, 1.0 + i // 10 % 10, 1.0 + i // 100])
for i in range(1100)
]

def test_rebuilds_an_existing_graph(self, temp_db):
_make_type(temp_db)
temp_db.command(
"sql",
"ALTER DATABASE `arcadedb.vectorIndex.inactivityRebuildTimeoutMs` 0",
)
_insert(temp_db, self.ROWS[:1000])
index = temp_db.create_vector_index(
"Doc", "embedding", dimensions=3, mutations_before_rebuild=1_000_000
)
before = index.get_stats()
assert before["graphNodeCount"] == 1000

_insert(temp_db, self.ROWS[1000:])
assert index.get_stats()["graphNodeCount"] == 1000

index.build_graph_now()

after = index.get_stats()
assert after["totalVectors"] == len(self.ROWS)
assert after["graphNodeCount"] == len(self.ROWS)
assert after["graphRebuildCount"] > before["graphRebuildCount"]


class TestWrapperAcrossRebuildAndDrop:
"""A ``VectorIndex`` held across SQL ``REBUILD INDEX`` keeps serving; across a drop it raises (#116).

Expand Down
Loading