Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
1fe1160
Tests: native GraphBatch.create_vertices sends columns and reads RIDs…
tae898 Nov 14, 2026
c8d2b87
Native: GraphBatch.create_vertices sends a chunk as columns and gets …
tae898 Nov 14, 2026
61c200d
Tests first: get_rid, get_vertex, get_edge and get_element of a row a…
tae898 Nov 14, 2026
2c64e8f
JVM: Result.get_rid, get_vertex, get_edge, get_element and Document.g…
tae898 Nov 14, 2026
6f44004
Tests: native async create_record hands records over in batches and e…
tae898 Nov 14, 2026
fd65da7
Native: async create_record hands records to the library as one colum…
tae898 Nov 14, 2026
ec9ebcf
Tests: the async create_record tests import numpy at the top, so a mi…
tae898 Nov 14, 2026
89c44d9
Tests: the type dump of the create_record parity test uses literal st…
tae898 Nov 14, 2026
0966a23
Tests: fix the stand-in and three expectations of the native GraphBat…
tae898 Nov 14, 2026
fa35ac4
Tests: the native create_record tests do not close the database's asy…
tae898 Nov 14, 2026
9b39edf
Native: a collected create_record is handed over after 50 ms with no …
tae898 Nov 14, 2026
c0b3aae
Native: create_vertices stores float32 array vectors after the other …
tae898 Nov 14, 2026
4f5e5fb
Tests: three native files that pinned the JSON route of create_vertic…
tae898 Nov 14, 2026
f15a1f9
Tests: the native async create_record parity test compares the stored…
tae898 Nov 14, 2026
449b65d
Tests: take tests/test_import_database.py from main (the post3 tag co…
tae898 Nov 14, 2026
84849b5
JVM: get_rid, get_vertex, get_edge, get_element and Document.get_rid …
tae898 Nov 14, 2026
44552d4
JVM: Document.get_rid stays on the plain path, no bridge call (refs #…
tae898 Nov 15, 2026
013da60
Native create_vertices: a chunk of fewer than 64 rows keeps the JSON …
tae898 Nov 15, 2026
6574a54
Native async create_record: the first 32 records of an executor go on…
tae898 Nov 15, 2026
1fd3ac5
Native create_vertices: the comment of the 64 row floor names the mic…
tae898 Nov 15, 2026
ae0623b
Tests: a def in the floor test, not an assigned lambda (ruff E731 of …
tae898 Nov 15, 2026
72d8acb
Native async create_record: a hand-over of fewer than 64 collected re…
tae898 Nov 15, 2026
2b2eed1
Tests: nosec B404 and B603 on the subprocess of the floor-from-enviro…
tae898 Nov 15, 2026
7854838
Native async create_record: the first 96 records of an executor go on…
tae898 Nov 15, 2026
b77f887
Native async create_record: the first 128 records of an executor go o…
tae898 Nov 15, 2026
ff062ae
Merge PR #390 (update to 2b2eed10cc) into post4/integration-v5-core3
tae898 Nov 15, 2026
43e1e52
Merge PR #393 (update to b77f887d44) into post4/integration-v5-core3
tae898 Nov 15, 2026
e83947e
Merge PR #395 (update to 44552d4a14) into post4/integration-v5-core3
tae898 Nov 15, 2026
9b984a8
Merge the 26.11.1.post4 release line into main
tae898 Nov 16, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
286 changes: 187 additions & 99 deletions native/src/main/java/org/humemai/nativelib/BulkOps.java

Large diffs are not rendered by default.

147 changes: 136 additions & 11 deletions native/src/main/java/org/humemai/nativelib/NativeExtras.java
Original file line number Diff line number Diff line change
Expand Up @@ -501,6 +501,42 @@ public static CCharPointer gbBin(final IsolateThread thread, final long h, final
return gbImpl(h, op, argsJson, buf.rawValue(), bufFloats);
}

/**
* create_vertices from columns, by pointer, as arcadedb_n_insert_colp takes them (see BulkOps.Table): header JSON {"n":rows,"cols":[[name,
* kind, dim],..],"last":[names]} (the columns stored after the others, see BulkOps.vertexRows) and an address table of nspec longs (BulkOps.SLOTS per column). The rows are built from the columns (BulkOps.vertexRows:
* the objects, and the property order, that the JSON rows of arcadedb_n_gb give) and go to the same GraphBatch.createVertices(type,
* Object[][]), so the transaction, the retries and the commit are the engine's and nothing differs from the JSON route but the transport.
* The RIDs do not come back as text: rids points at ridLongs int64 slots of the caller's (at least 2 per row), and row i gets bucket id
* at slot 2i and position at slot 2i + 1. Returns {"ok":true,"n":rows}. The caller keeps every buffer alive until the call returns.
*/
@CEntryPoint(name = "arcadedb_n_gb_colp")
public static CCharPointer gbColp(final IsolateThread thread, final long h, final CCharPointer type, final CCharPointer header,
final CCharPointer spec, final int nspec, final CCharPointer rids, final long ridLongs) {
try {
final BatchEntry entry = BATCHES.get(h);
if (entry == null)
throw new IllegalArgumentException("unknown GraphBatch " + h);
if (nspec < 0 || nspec % BulkOps.SLOTS != 0)
throw new IllegalArgumentException("address table of " + nspec + " entries is not a multiple of " + BulkOps.SLOTS);
final long[] table = new long[nspec];
for (int i = 0; i < nspec; i++)
table[i] = ((Pointer) spec).readLong(WordFactory.signed(8L * i));
final Map<String, Object> head = args(header);
final BulkOps.Table columns = BulkOps.Table.of(head, table);
if (rids.isNull() || ridLongs < 2L * columns.n)
throw new IllegalArgumentException("the RID buffer holds " + ridLongs + " int64 slots, " + 2L * columns.n + " needed");
final RID[] created = entry.batch().createVertices(ArcadeNative.str(type), BulkOps.vertexRows(columns, head.get("last")));
final Pointer out = (Pointer) rids;
for (int i = 0; i < created.length; i++) {
out.writeLong(WordFactory.signed(16L * i), created[i].getBucketId());
out.writeLong(WordFactory.signed(16L * i + 8L), created[i].getPosition());
}
return ArcadeNative.out("{\"ok\":true,\"n\":" + created.length + "}");
} catch (final Throwable t) {
return ArcadeNative.err(t);
}
}

@SuppressWarnings("unchecked")
private static CCharPointer gbImpl(final long h, final CCharPointer op, final CCharPointer argsJson, final long buf,
final long bufFloats) {
Expand Down Expand Up @@ -614,6 +650,78 @@ void drop() {
}
}

/**
* The callback of a record that an async writer stored: Python gets the stored record (its RID included) under the callback id, or
* nothing when the record has no callback.
*/
private static com.arcadedb.database.async.NewRecordCallback recordCallback(final long cb) {
if (cb == 0)
return null;
return rec -> {
try {
final StringBuilder sb = new StringBuilder("\"rec\":");
NativeCodec.append(sb, rec);
fire("record callback", cb, event("record", sb.toString()));
} catch (final Throwable e) {
swallowed("record callback", e); // the writer goes on
}
};
}

/**
* The callbacks of the records of one "create_records" batch (callback id and error callback id per record, 0 for none). It is the
* error callback that BulkOps gets for the frame; BulkOps hands record i to {@link #submit} when it is one of these, so that every
* record has its own callbacks, as a "create_record" call has.
*/
static final class RecordHooks implements ErrorCallback {
private final long[] cbs;
private final long[] errs;
/** The records handed to the writers so far: after a failure, the number of records that the writers hold. */
int handed;

RecordHooks(final long[] cbs, final long[] errs) {
this.cbs = cbs;
this.errs = errs;
}

/** Not used: every record is handed over with its own error callback. */
@Override
public void call(final Throwable t) {
swallowed("create_records error callback", t);
}

void submit(final DatabaseAsyncExecutor ex, final MutableDocument doc, final int i) {
final com.arcadedb.database.async.NewRecordCallback ok = recordCallback(cbs[i]);
if (errs[i] == 0)
ex.createRecord(doc, ok);
else
ex.createRecord(doc, ok, errorCallback(errs[i]));
handed = i + 1;
}
}

/** The address table of arcadedb_n_async_bin as a long[] of BulkOps.SLOTS entries per column (the layout of arcadedb_n_insert_colp). */
private static long[] columnTable(final long table, final int nTable) {
if (table == 0 || nTable <= 0 || nTable % BulkOps.SLOTS != 0)
throw new IllegalArgumentException("address table of " + nTable + " entries is not a multiple of " + BulkOps.SLOTS);
final Pointer t = WordFactory.pointer(table);
final long[] spec = new long[nTable];
for (int i = 0; i < nTable; i++)
spec[i] = t.readLong(WordFactory.signed(8L * i));
return spec;
}

@SuppressWarnings("unchecked")
private static long[] callbackIds(final Object list, final int n, final String what) {
final List<Object> in = (List<Object>) list;
if (in == null || in.size() != n)
throw new IllegalArgumentException("\"" + what + "\" needs one entry per record (" + n + ")");
final long[] out = new long[n];
for (int i = 0; i < n; i++)
out[i] = ((Number) in.get(i)).longValue();
return out;
}

private static String replyRows(final ResultSet rs) {
final StringBuilder sb = new StringBuilder("{\"ok\":true,\"rs\":0,");
NativeCodec.appendBatch(sb, rs, Integer.MAX_VALUE);
Expand Down Expand Up @@ -673,8 +781,8 @@ private static boolean isVertexType(final Database db, final String typeName) {
}

/**
* ops: get | set {key,value} | wait {timeout_ms} | kill | create_record {type, kind, props, cb, err} | command / query {lang, text, params,
* cb, err} | on_ok {cb} | on_error {cb} | append_samples {type, ts, cols} | insert_parallel {type, rows} | failures {fid}.
* ops: get | set {key,value} | wait {timeout_ms} | kill | create_record {type, kind, props, cb, err} | create_records {type, kind, n, commit,
* cols, cbs, errs} (arcadedb_n_async_bin only: a frame) | command / query {lang, text, params, cb, err} | on_ok {cb} | on_error {cb} | append_samples {type, ts, cols} | insert_parallel {type, rows} | failures {fid}.
*/
@CEntryPoint(name = "arcadedb_n_async")
public static CCharPointer async(final IsolateThread thread, final long dbh, final CCharPointer op, final CCharPointer argsJson) {
Expand Down Expand Up @@ -781,20 +889,37 @@ private static CCharPointer asyncImpl(final long dbh, final CCharPointer op, fin
final MutableDocument doc = "v".equals(a.get("kind")) ? db.newVertex(type) : db.newDocument(type);
for (final Map.Entry<String, Object> e : ((Map<String, Object>) a.get("props")).entrySet())
doc.set(e.getKey(), e.getValue());
final com.arcadedb.database.async.NewRecordCallback ok = cb == 0 ? null : rec -> {
try {
final StringBuilder sb = new StringBuilder("\"rec\":");
NativeCodec.append(sb, rec);
fire("record callback", cb, event("record", sb.toString()));
} catch (final Throwable e) {
swallowed("record callback", e); // the writer goes on
}
};
final com.arcadedb.database.async.NewRecordCallback ok = recordCallback(cb);
if (errCb == 0)
ex.createRecord(doc, ok);
else
ex.createRecord(doc, ok, errorCallback(errCb));
}
case "create_records" -> {
// a frame of records of one type and kind, each with its own callbacks (cbs, errs: callback ids, 0 for none), through BulkOps' column
// reader. The frame and the type are checked before the first record is handed over (an exception then: nothing was queued). A
// record that fails while the batch is handed over ends the call with {"ok":true,"n":k,"err":..}: k records are the writers'
// now and the record k is refused with that error, so the caller can give it to its error callback and send the rest again.
final String type = (String) a.get("type");
final boolean vertex = "v".equals(a.get("kind"));
final Object schemaType = db.getSchema().getType(type);
if (schemaType instanceof EdgeType)
throw new IllegalArgumentException("Cannot bulk-insert into '" + type
+ "' because it is an edge type; load the vertices, then create the edges (graph_batch().new_edges() or CREATE EDGE)");
if (vertex != (schemaType instanceof VertexType))
throw new IllegalArgumentException("'" + type + "' is " + (vertex ? "not a vertex type" : "a vertex type")
+ ", but its records were created as " + (vertex ? "vertices" : "documents"));
final int n = intOf(a.get("n"), -1);
final RecordHooks hooks = new RecordHooks(callbackIds(a.get("cbs"), n, "cbs"), callbackIds(a.get("errs"), n, "errs"));
try {
final long handed = BulkOps.handColumns(db, type, a, columnTable(table, nTable), hooks);
return ArcadeNative.out("{\"ok\":true,\"n\":" + handed + "}");
} catch (final IllegalStateException e) {
if (e.getMessage() == null || !e.getMessage().startsWith("after handing ") || e.getCause() == null)
throw e;
return ArcadeNative.out("{\"ok\":true,\"n\":" + hooks.handed + "," + errorBody(e.getCause()) + "}");
}
}
case "command", "query" -> {
final boolean command = "command".equals(ArcadeNative.str(op));
final com.arcadedb.database.async.AsyncResultsetCallback callback = cb == 0 && errCb == 0 ? null : new com.arcadedb.database.async.AsyncResultsetCallback() {
Expand Down
55 changes: 55 additions & 0 deletions src/arcadedb/_native/bulk.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,3 +95,58 @@ def insert_rows_columnar(db, type_name, chunk):
rows, cols = packed
_send(db, type_name, cols, rows, 0, f"Failed to bulk-insert into '{type_name}'")
return True


def rids_of(out):
"""The RID texts (``#bucket:position``, the ``RID.toString`` text) of an int64 array of ``(bucket, position)`` pairs, in row order."""
pairs = out.reshape(-1, 2)
return list(map("#{}:{}".format, pairs[:, 0].tolist(), pairs[:, 1].tolist()))


def create_vertices_columns(gb, type_name, rows, declared, prefix):
"""One ``GraphBatch.create_vertices`` chunk (a list of dicts) through ``arcadedb_n_gb_colp``: the rows cross as columns by address and
the RIDs come back in an int64 buffer, one crossing and one transaction, as the JSON route has it. Returns the list of RID texts, or None
when the chunk needs the JSON route (nothing was sent then): rows that do not all have the same keys in the same order, a value no column
holds (a nested list or map, a mixed-type column, a set, a NaT, a numpy scalar, an integer past 64 bits, a lone surrogate), an empty chunk or a
chunk of fewer rows than ``lib.GB_COLP_MIN_ROWS`` (64 by default: a handful of rows does not repay the fixed cost of the columnar entry).

``declared``: a :class:`columnar.Declared` of the type, read once by the caller for all chunks of a call.
"""
if not (lib.GB_COLP and _usable()) or len(rows) < lib.GB_COLP_MIN_ROWS:
return None # a chunk below the floor keeps the JSON route of post3 (lib.GB_COLP_MIN_ROWS), as does a chunk the columns cannot carry
columns = columnar.rows_to_columns(rows, same_order=True)
if columns is None:
return None
packed = columnar.pack(
columns, declared, exact_nanos=True
) # the JSON route keeps the nanoseconds of a Timestamp
if packed is None:
return None
count, cols = packed
# a column of float32 arrays is stored after the other properties of a row: the old route took those vectors out of the JSON rows and
# appended them (NativeExtras.withVectors); the other columns keep the order of a HashMap of their names, as the JSON rows do
last = [
c.name
for c in cols
if c.kind == "f" and isinstance(columns[c.name][0], columnar._np.ndarray)
]
spec = columnar.spec_of(cols)
out = columnar._np.empty(2 * count, dtype=columnar._np.int64)
from .database import _call

reply = _call(
prefix,
"arcadedb_n_gb_colp",
gb._gb,
lib.b(type_name),
lib.b(columnar.header_of(count, 0, cols, last=last)),
spec,
len(spec),
out.ctypes.data,
out.size,
) # `cols` and `out` hold the memory the addresses point into until the call returns
if reply["n"] != count:
raise ArcadeDBError(
f"{prefix}: the library created {reply['n']} vertices for {count} rows"
)
return rids_of(out)
30 changes: 23 additions & 7 deletions src/arcadedb/_native/columnar.py
Original file line number Diff line number Diff line change
Expand Up @@ -215,7 +215,7 @@ def _epoch_micros(value):
return seconds * 1_000_000 + value.microsecond


def _list_column(name, values, declared):
def _list_column(name, values, declared, exact_nanos=False):
"""A list or tuple of Python values as a column: one type (None allowed) or a list of equal-length float arrays; else None."""
n = len(values)
types = set(map(type, values))
Expand Down Expand Up @@ -255,6 +255,12 @@ def _list_column(name, values, declared):
if issubclass(only, datetime):
if only.__name__ == "NaTType":
return None # the JSON path stores it as a null
if (
exact_nanos
and only is not datetime
and any(getattr(v, "nanosecond", 0) for v in values if v is not None)
):
return None # a pandas Timestamp with nanoseconds, for a caller whose JSON route keeps them (insert_many's keeps microseconds)
try:
ticks = [0 if v is None else _epoch_micros(v) for v in values]
except (OverflowError, ValueError):
Expand Down Expand Up @@ -407,7 +413,7 @@ def _arrow_column(name, arr, declared):
return None


def pack(columns, declared):
def pack(columns, declared, exact_nanos=False):
"""``(rows, [Packed])`` for a dict of name -> column, or None when any column cannot be laid out or the columns differ in length.

A column is a numpy array (1-D ints, floats and bools, or a 2-D float32 matrix), a list or tuple of Python values of one type (None
Expand All @@ -425,14 +431,14 @@ def pack(columns, declared):
elif isinstance(values, _np.ndarray):
if values.dtype.kind in "OUS":
col = (
_list_column(name, values.tolist(), declared)
_list_column(name, values.tolist(), declared, exact_nanos)
if values.ndim == 1
else None
)
else:
col = _numpy_column(name, values, declared)
elif isinstance(values, (list, tuple)) and values:
col = _list_column(name, values, declared)
col = _list_column(name, values, declared, exact_nanos)
else:
col = None # a pandas Series, a generator, an empty list: the row path handles them
if col is None:
Expand All @@ -454,7 +460,7 @@ def spec_of(packed):
return (ctypes.c_int64 * len(flat))(*flat)


def header_of(rows, commit_every, packed, fid=None):
def header_of(rows, commit_every, packed, fid=None, last=()):
"""The JSON header text: row count, commit interval and ``[name, kind, dim]`` per column; with a failure token ``fid`` it also says
``"parallel": true``, which hands the records to the async executor (the image then ignores the commit interval and counts the
failed records on that token).
Expand All @@ -466,14 +472,22 @@ def header_of(rows, commit_every, packed, fid=None):
"commit": min(int(commit_every), 2**31 - 1),
"cols": [[c.name, c.kind, c.dim] for c in packed],
}
if (
last
): # create_vertices: the columns the old route appended to the end of each row (float32 array vectors)
header["last"] = list(last)
if fid is not None:
header["parallel"] = True
header["fid"] = fid
return json.dumps(header)


def rows_to_columns(rows):
"""A list of dicts with the same keys as a dict of name -> list of values, or None (no rows, different keys, a non-str key)."""
def rows_to_columns(rows, same_order=False):
"""A list of dicts with the same keys as a dict of name -> list of values, or None (no rows, different keys, a non-str key).

``same_order``: every row must also list its keys in the order of the first row (a row with the same keys in another order gives
None), for a caller whose stored property order follows the key order of its rows.
"""
if not rows:
return None
try:
Expand All @@ -482,6 +496,8 @@ def rows_to_columns(rows):
return None
if set(map(len, rows)) != {len(keys)}:
return None
if same_order and not all(map(keys.__eq__, map(list, rows))):
return None
return {key: [row[key] for row in rows] for key in keys}
except (
KeyError,
Expand Down
Loading
Loading