` | RRF with uniform weights |
---
diff --git a/pom.xml b/pom.xml
index ba1dc76..e592ac5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -6,7 +6,7 @@
io.endee
endee-java-client
- 1.0.2-SNAPSHOT
+ 2.0.0-SNAPSHOT
jar
Endee Java Client
diff --git a/src/main/java/io/endee/client/Collection.java b/src/main/java/io/endee/client/Collection.java
new file mode 100644
index 0000000..a0651a2
--- /dev/null
+++ b/src/main/java/io/endee/client/Collection.java
@@ -0,0 +1,842 @@
+package io.endee.client;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import io.endee.client.exception.EndeeApiException;
+import io.endee.client.exception.EndeeException;
+import io.endee.client.types.*;
+import io.endee.client.util.CryptoUtils;
+import io.endee.client.util.JsonUtils;
+import io.endee.client.util.MessagePackUtils;
+import io.endee.client.util.ValidationUtils;
+import java.io.IOException;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.time.Duration;
+import java.util.*;
+import java.util.stream.Collectors;
+
+/**
+ * Collection client for Endee-DB vector operations (v2 API).
+ *
+ * Obtain via {@link Endee#getCollection(String)}.
+ */
+public class Collection {
+ private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(30);
+ private static final int MAX_BATCH_SIZE = 10_000;
+ private static final int MAX_TOP_K = 4096;
+ private static final int MAX_EF_SEARCH = 1024;
+ private static final int MAX_FILTER_BOOST_PERCENTAGE = 100;
+ private static final int MIN_PREFILTER_THRESHOLD = 1_000;
+ private static final int MAX_PREFILTER_THRESHOLD = 1_000_000;
+ private static final String NORMS_KEY = "internal_";
+
+ private final String name;
+ private final String token;
+ private final String baseUrl;
+ private final HttpClient httpClient;
+ private final ObjectMapper objectMapper;
+ private List> fields;
+
+ public Collection(String name, String token, String baseUrl, Map metadata) {
+ this.name = name;
+ this.token = token;
+ this.baseUrl = baseUrl;
+ this.objectMapper = new ObjectMapper();
+
+ @SuppressWarnings("unchecked")
+ List> fieldsList =
+ (List>) metadata.getOrDefault("fields", List.of());
+ this.fields = fieldsList;
+
+ this.httpClient =
+ HttpClient.newBuilder()
+ .version(HttpClient.Version.HTTP_2)
+ .connectTimeout(DEFAULT_TIMEOUT)
+ .build();
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ /** Returns field metadata: name → {type, space_type, dimension}. */
+ private Map> fieldMap() {
+ Map> idx = new LinkedHashMap<>();
+ for (Map f : fields) {
+ @SuppressWarnings("unchecked")
+ Map params =
+ (Map) f.getOrDefault("params", Map.of());
+ Map entry = new HashMap<>();
+ entry.put("type", f.getOrDefault("type", "vector"));
+ entry.put("space_type", params.getOrDefault("space_type", "cosine"));
+ entry.put("dimension", params.getOrDefault("dimension", 0));
+ idx.put((String) f.get("name"), entry);
+ }
+ return idx;
+ }
+
+ // ── upsert ──────────────────────────────────────────────────────────────────
+
+ /**
+ * Upserts objects into the collection.
+ *
+ * @param objects list of objects to upsert (max 10,000)
+ * @return server response
+ */
+ public Map upsert(List objects) {
+ if (objects == null || objects.isEmpty()) {
+ throw new IllegalArgumentException("Must provide at least one object to upsert");
+ }
+ if (objects.size() > MAX_BATCH_SIZE) {
+ throw new IllegalArgumentException(
+ "Cannot upsert more than " + MAX_BATCH_SIZE + " objects at a time");
+ }
+
+ List ids =
+ objects.stream().map(ObjectItem::getId).collect(Collectors.toList());
+ ValidationUtils.validateObjectIds(ids);
+
+ Map> fMap = fieldMap();
+ List wireObjects = new ArrayList<>();
+
+ for (ObjectItem item : objects) {
+ String filterStr = "";
+ if (item.getFilter() != null && !item.getFilter().isEmpty()) {
+ filterStr = JsonUtils.toJson(item.getFilter());
+ }
+
+ Map vectors = new LinkedHashMap<>();
+ Map sparses = new LinkedHashMap<>();
+ Map multiVectors = new LinkedHashMap<>();
+ Map norms = new LinkedHashMap<>();
+
+ if (item.getFields() != null) {
+ for (Map.Entry fe : item.getFields().entrySet()) {
+ String fname = fe.getKey();
+ Object fdata = fe.getValue();
+ Map cfg = fMap.get(fname);
+ if (cfg == null) {
+ throw new IllegalArgumentException(
+ "Unknown field '" + fname + "'. Collection fields: " + fMap.keySet());
+ }
+
+ String ftype = (String) cfg.get("type");
+ String space = (String) cfg.getOrDefault("space_type", "cosine");
+ int dim = cfg.get("dimension") instanceof Number ? ((Number) cfg.get("dimension")).intValue() : 0;
+
+ if ("vector".equals(ftype)) {
+ double[] vec = (double[]) fdata;
+ validateVectorValues(vec, item.getId());
+ double[] normalized = normalizeDense(vec, space);
+ double norm = computeNorm(vec);
+ if (dim > 0 && vec.length != dim) {
+ throw new IllegalArgumentException(
+ "Field '" + fname + "': expected dimension " + dim + ", got " + vec.length);
+ }
+ vectors.put(fname, normalized);
+ if ("cosine".equals(space)) {
+ norms.put(fname, norm);
+ }
+ } else if ("sparse".equals(ftype)) {
+ SparseData sd = (SparseData) fdata;
+ sparses.put(fname, new Object[] {sd.getIndices(), sd.getValues()});
+ } else if ("multi_vector".equals(ftype)) {
+ double[][] vecs = (double[][]) fdata;
+ double[][] normalizedVecs = new double[vecs.length][];
+ List vecNorms = new ArrayList<>();
+ for (int i = 0; i < vecs.length; i++) {
+ validateVectorValues(vecs[i], item.getId());
+ if (dim > 0 && vecs[i].length != dim) {
+ throw new IllegalArgumentException(
+ "Field '"
+ + fname
+ + "': every multi_vector must have dimension "
+ + dim);
+ }
+ normalizedVecs[i] = normalizeDense(vecs[i], space);
+ if ("cosine".equals(space)) {
+ vecNorms.add(computeNorm(vecs[i]));
+ }
+ }
+ multiVectors.put(fname, normalizedVecs);
+ if ("cosine".equals(space) && !vecNorms.isEmpty()) {
+ norms.put(fname, vecNorms);
+ }
+ } else {
+ throw new IllegalArgumentException(
+ "Field '" + fname + "' has unknown type '" + ftype + "'");
+ }
+ }
+ }
+
+ // Build meta with norms
+ Map rawMeta =
+ item.getMeta() != null ? new HashMap<>(item.getMeta()) : new HashMap<>();
+ if (!norms.isEmpty()) {
+ rawMeta.put(NORMS_KEY, norms);
+ }
+ byte[] metaBytes = CryptoUtils.jsonZip(rawMeta);
+
+ wireObjects.add(
+ new Object[] {item.getId(), metaBytes, filterStr, vectors, sparses, multiVectors});
+ }
+
+ byte[] payload = MessagePackUtils.packObjects(wireObjects);
+
+ try {
+ HttpRequest request =
+ buildPostMsgpackRequest("/collection/" + name + "/objects", payload);
+ HttpResponse response =
+ httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+
+ if (response.statusCode() != 200) {
+ EndeeApiException.raiseException(response.statusCode(), response.body());
+ }
+
+ @SuppressWarnings("unchecked")
+ Map result = objectMapper.readValue(response.body(), Map.class);
+ return result;
+ } catch (IOException | InterruptedException e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ throw new EndeeException("Failed to upsert objects", e);
+ }
+ }
+
+ // ── search ──────────────────────────────────────────────────────────────────
+
+ /**
+ * Searches the collection across one or more fields.
+ *
+ * @param queryFields field_name → {query, limit?, ef_search?}
+ * @param filter optional filter conditions
+ * @param efSearch default ef_search (max 1024)
+ * @param prefilterThreshold optional prefilter cardinality threshold (1000-1000000)
+ * @param boostPercentage optional filter boost percentage (0-100)
+ * @return per-field results: field_name → list of hits
+ */
+ public Map> search(
+ Map> queryFields,
+ List> filter,
+ int efSearch,
+ Integer prefilterThreshold,
+ Integer boostPercentage) {
+
+ if (queryFields == null || queryFields.isEmpty()) {
+ throw new IllegalArgumentException("search requires at least one field");
+ }
+ if (efSearch < 1 || efSearch > MAX_EF_SEARCH) {
+ throw new IllegalArgumentException("ef_search must be between 1 and " + MAX_EF_SEARCH);
+ }
+ if (prefilterThreshold != null
+ && (prefilterThreshold < MIN_PREFILTER_THRESHOLD
+ || prefilterThreshold > MAX_PREFILTER_THRESHOLD)) {
+ throw new IllegalArgumentException(
+ "prefilter_cardinality_threshold must be between "
+ + MIN_PREFILTER_THRESHOLD
+ + " and "
+ + MAX_PREFILTER_THRESHOLD);
+ }
+ if (boostPercentage != null
+ && (boostPercentage < 0 || boostPercentage > MAX_FILTER_BOOST_PERCENTAGE)) {
+ throw new IllegalArgumentException(
+ "filter_boost_percentage must be between 0 and " + MAX_FILTER_BOOST_PERCENTAGE);
+ }
+
+ Map> fMap = fieldMap();
+ List> fieldsArray = new ArrayList<>();
+ Map fieldLimits = new LinkedHashMap<>();
+
+ for (Map.Entry> entry : queryFields.entrySet()) {
+ String fname = entry.getKey();
+ Map fdata = new LinkedHashMap<>(entry.getValue());
+
+ // Normalize query vectors for cosine fields
+ Object query = fdata.get("query");
+ Map fld = fMap.get(fname);
+ if (fld != null) {
+ String ftype = (String) fld.get("type");
+ String space = (String) fld.getOrDefault("space_type", "cosine");
+ if ("vector".equals(ftype) && "cosine".equals(space) && query instanceof double[]) {
+ fdata.put("query", normalizeDense((double[]) query, space));
+ } else if ("multi_vector".equals(ftype)
+ && "cosine".equals(space)
+ && query instanceof double[][]) {
+ double[][] qVecs = (double[][]) query;
+ double[][] normalized = new double[qVecs.length][];
+ for (int i = 0; i < qVecs.length; i++) {
+ normalized[i] = normalizeDense(qVecs[i], space);
+ }
+ fdata.put("query", normalized);
+ }
+ }
+
+ // Resolve limit
+ int limit = 10;
+ if (fdata.containsKey("limit") && fdata.get("limit") instanceof Number) {
+ limit = ((Number) fdata.get("limit")).intValue();
+ if (limit < 1 || limit > MAX_TOP_K) {
+ throw new IllegalArgumentException(
+ "Search field '" + fname + "': limit must be between 1 and " + MAX_TOP_K);
+ }
+ }
+ fieldLimits.put(fname, limit);
+
+ // Build entry
+ Map fieldEntry = new LinkedHashMap<>(fdata);
+ fieldEntry.put("limit", limit);
+ if (!fieldEntry.containsKey("ef_search")) {
+ fieldEntry.put("ef_search", efSearch);
+ }
+
+ // Convert SparseData query to map format for JSON
+ Object q = fieldEntry.get("query");
+ if (q instanceof SparseData sd) {
+ Map sparseQuery = new LinkedHashMap<>();
+ sparseQuery.put("indices", sd.getIndices());
+ sparseQuery.put("values", sd.getValues());
+ fieldEntry.put("query", sparseQuery);
+ }
+
+ fieldsArray.add(Map.of(fname, fieldEntry));
+ }
+
+ Map payload = new LinkedHashMap<>();
+ payload.put("fields", fieldsArray);
+ if (filter != null) {
+ payload.put("filter", filter);
+ }
+ if (prefilterThreshold != null || boostPercentage != null) {
+ Map filterParams = new LinkedHashMap<>();
+ filterParams.put(
+ "prefilter_threshold", prefilterThreshold != null ? prefilterThreshold : 10_000);
+ filterParams.put("boost_percentage", boostPercentage != null ? boostPercentage : 0);
+ payload.put("filter_params", filterParams);
+ }
+
+ try {
+ String jsonBody = JsonUtils.toJson(payload);
+ HttpRequest request = buildPostJsonRequest("/collection/" + name + "/search", jsonBody);
+ HttpResponse response =
+ httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray());
+
+ if (response.statusCode() != 200) {
+ EndeeApiException.raiseException(response.statusCode(), new String(response.body()));
+ }
+
+ Object[] decoded = MessagePackUtils.unpackSearchResponse(response.body());
+ @SuppressWarnings("unchecked")
+ Map objectsMap = (Map) decoded[0];
+ @SuppressWarnings("unchecked")
+ Map> resultsMap = (Map>) decoded[1];
+
+ Map> perField = new LinkedHashMap<>();
+ for (String fname : queryFields.keySet()) {
+ List hits = resultsMap.getOrDefault(fname, List.of());
+ int limit = fieldLimits.getOrDefault(fname, 10);
+ List fieldHits = new ArrayList<>();
+ for (Object[] hit : hits) {
+ if (fieldHits.size() >= limit) break;
+ int intId = (Integer) hit[0];
+ double score = (Double) hit[1];
+ Object[] objMeta = objectsMap.get(intId);
+ SearchHit sh = new SearchHit();
+ if (objMeta != null) {
+ sh.setId((String) objMeta[0]);
+ Map meta = CryptoUtils.jsonUnzip((byte[]) objMeta[1]);
+ if (meta != null) {
+ meta.remove(NORMS_KEY);
+ }
+ sh.setMeta(meta);
+ String filterStr = (String) objMeta[2];
+ if (filterStr != null && !filterStr.isEmpty()) {
+ @SuppressWarnings("unchecked")
+ Map parsedFilter = JsonUtils.fromJson(filterStr, Map.class);
+ sh.setFilter(parsedFilter);
+ }
+ } else {
+ sh.setId(String.valueOf(intId));
+ sh.setMeta(Map.of());
+ }
+ sh.setSimilarity(score);
+ fieldHits.add(sh);
+ }
+ perField.put(fname, fieldHits);
+ }
+
+ return perField;
+ } catch (IOException | InterruptedException e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ throw new EndeeException("Failed to search collection", e);
+ }
+ }
+
+ /** Convenience overload with defaults: efSearch=128, no filter tuning. */
+ public Map> search(
+ Map> queryFields,
+ List> filter) {
+ return search(queryFields, filter, 128, null, null);
+ }
+
+ /** Convenience overload: no filter. */
+ public Map> search(
+ Map> queryFields) {
+ return search(queryFields, null, 128, null, null);
+ }
+
+ // ── get objects ──────────────────────────────────────────────────────────────
+
+ /**
+ * Fetches full objects by ID.
+ *
+ * @param ids list of object IDs
+ * @return list of objects with vectors, sparses, multi_vectors
+ */
+ public List getObjects(List ids) {
+ if (ids == null || ids.isEmpty()) {
+ throw new IllegalArgumentException("getObjects requires a non-empty list of ids");
+ }
+
+ try {
+ String jsonBody = JsonUtils.toJson(Map.of("ids", ids));
+ HttpRequest request =
+ buildPostJsonRequest("/collection/" + name + "/objects/query", jsonBody);
+ HttpResponse response =
+ httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray());
+
+ if (response.statusCode() != 200) {
+ EndeeApiException.raiseException(response.statusCode(), new String(response.body()));
+ }
+
+ List batch = MessagePackUtils.unpackObjectBatch(response.body());
+ List results = new ArrayList<>();
+
+ for (Object[] obj : batch) {
+ ObjectInfo info = new ObjectInfo();
+ info.setId((String) obj[0]);
+
+ Map meta = CryptoUtils.jsonUnzip((byte[]) obj[1]);
+ @SuppressWarnings("unchecked")
+ Map normsMap =
+ meta != null ? (Map) meta.remove(NORMS_KEY) : null;
+ info.setMeta(meta != null ? meta : Map.of());
+
+ String filterStr = (String) obj[2];
+ if (filterStr != null && !filterStr.isEmpty()) {
+ @SuppressWarnings("unchecked")
+ Map parsedFilter = JsonUtils.fromJson(filterStr, Map.class);
+ info.setFilter(parsedFilter);
+ } else {
+ info.setFilter(Map.of());
+ }
+
+ // Vectors — reconstruct originals using norms
+ @SuppressWarnings("unchecked")
+ Map vectors = (Map) obj[3];
+ if (normsMap != null && vectors != null) {
+ for (Map.Entry ve : vectors.entrySet()) {
+ Object normVal = normsMap.get(ve.getKey());
+ if (normVal instanceof Number n) {
+ double norm = n.doubleValue();
+ double[] vec = ve.getValue();
+ for (int i = 0; i < vec.length; i++) {
+ vec[i] *= norm;
+ }
+ }
+ }
+ }
+ info.setVectors(vectors != null ? vectors : Map.of());
+
+ // Sparses
+ @SuppressWarnings("unchecked")
+ Map sparsesRaw = (Map) obj[4];
+ Map sparses = new LinkedHashMap<>();
+ if (sparsesRaw != null) {
+ for (Map.Entry se : sparsesRaw.entrySet()) {
+ sparses.put(
+ se.getKey(),
+ new SparseData((int[]) se.getValue()[0], (double[]) se.getValue()[1]));
+ }
+ }
+ info.setSparses(sparses);
+
+ // Multi-vectors — reconstruct originals using norms
+ @SuppressWarnings("unchecked")
+ Map multiVecs = (Map) obj[5];
+ if (normsMap != null && multiVecs != null) {
+ for (Map.Entry mve : multiVecs.entrySet()) {
+ Object normVal = normsMap.get(mve.getKey());
+ if (normVal instanceof List normsList) {
+ double[][] vecs = mve.getValue();
+ for (int i = 0; i < vecs.length && i < normsList.size(); i++) {
+ double n = ((Number) normsList.get(i)).doubleValue();
+ for (int j = 0; j < vecs[i].length; j++) {
+ vecs[i][j] *= n;
+ }
+ }
+ }
+ }
+ }
+ info.setMultiVectors(multiVecs != null ? multiVecs : Map.of());
+
+ results.add(info);
+ }
+
+ return results;
+ } catch (IOException | InterruptedException e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ throw new EndeeException("Failed to get objects", e);
+ }
+ }
+
+ // ── delete ──────────────────────────────────────────────────────────────────
+
+ /** Deletes a single object by ID. */
+ public Map deleteObject(String id) {
+ try {
+ HttpRequest request =
+ buildDeleteRequest("/collection/" + name + "/objects/" + id);
+ HttpResponse response =
+ httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+
+ if (response.statusCode() != 200) {
+ EndeeApiException.raiseException(response.statusCode(), response.body());
+ }
+
+ @SuppressWarnings("unchecked")
+ Map result = objectMapper.readValue(response.body(), Map.class);
+ return result;
+ } catch (IOException | InterruptedException e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ throw new EndeeException("Failed to delete object", e);
+ }
+ }
+
+ /** Deletes objects matching a filter. */
+ public Map deleteByFilter(List> filter) {
+ if (filter == null || filter.isEmpty()) {
+ throw new IllegalArgumentException(
+ "filter must be a non-empty array, e.g. [{'field': {'$op': value}}]");
+ }
+
+ try {
+ String jsonBody = JsonUtils.toJson(Map.of("filter", filter));
+ HttpRequest request =
+ buildDeleteJsonRequest("/collection/" + name + "/objects", jsonBody);
+ HttpResponse response =
+ httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+
+ if (response.statusCode() != 200) {
+ EndeeApiException.raiseException(response.statusCode(), response.body());
+ }
+
+ @SuppressWarnings("unchecked")
+ Map result = objectMapper.readValue(response.body(), Map.class);
+ return result;
+ } catch (IOException | InterruptedException e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ throw new EndeeException("Failed to delete by filter", e);
+ }
+ }
+
+ // ── update filters ──────────────────────────────────────────────────────────
+
+ /** Updates filter tags on existing objects. */
+ public Map updateFilters(List updates) {
+ if (updates == null || updates.isEmpty()) {
+ throw new IllegalArgumentException("updates must be a non-empty list");
+ }
+
+ List> payload = new ArrayList<>();
+ for (UpdateFilterParams update : updates) {
+ Map entry = new HashMap<>();
+ entry.put("id", update.getId());
+ entry.put("filter", update.getFilter() != null ? update.getFilter() : Map.of());
+ payload.add(entry);
+ }
+
+ try {
+ String jsonBody = JsonUtils.toJson(Map.of("updates", payload));
+ HttpRequest request =
+ buildPostJsonRequest("/collection/" + name + "/filters", jsonBody);
+ HttpResponse response =
+ httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+
+ if (response.statusCode() != 200) {
+ EndeeApiException.raiseException(response.statusCode(), response.body());
+ }
+
+ @SuppressWarnings("unchecked")
+ Map result = objectMapper.readValue(response.body(), Map.class);
+ return result;
+ } catch (IOException | InterruptedException e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ throw new EndeeException("Failed to update filters", e);
+ }
+ }
+
+ // ── describe ────────────────────────────────────────────────────────────────
+
+ /** Fetches collection metadata from the server. */
+ public Map describe() {
+ try {
+ HttpRequest request = buildGetRequest("/collection/" + name);
+ HttpResponse response =
+ httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+
+ if (response.statusCode() != 200) {
+ EndeeApiException.raiseException(response.statusCode(), response.body());
+ }
+
+ @SuppressWarnings("unchecked")
+ Map result = objectMapper.readValue(response.body(), Map.class);
+
+ // Update local fields metadata
+ @SuppressWarnings("unchecked")
+ List> updatedFields =
+ (List>) result.getOrDefault("fields", List.of());
+ this.fields = updatedFields;
+
+ return result;
+ } catch (IOException | InterruptedException e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ throw new EndeeException("Failed to describe collection", e);
+ }
+ }
+
+ // ── rebuild ─────────────────────────────────────────────────────────────────
+
+ /**
+ * Rebuilds HNSW graphs for one or more fields (async).
+ *
+ * @param fieldSpecs list of field specs, e.g. [{"field": "embedding", "M": 20, "ef_con": 200}]
+ */
+ public Map rebuild(List> fieldSpecs) {
+ if (fieldSpecs == null || fieldSpecs.isEmpty()) {
+ throw new IllegalArgumentException("rebuild requires a non-empty list of field specs");
+ }
+ for (Map spec : fieldSpecs) {
+ if (!spec.containsKey("field") || spec.get("field") == null) {
+ throw new IllegalArgumentException("Each field spec must include a 'field' name");
+ }
+ }
+
+ try {
+ String jsonBody = JsonUtils.toJson(Map.of("fields", fieldSpecs));
+ HttpRequest request = buildPostJsonRequest("/collection/" + name + "/rebuild", jsonBody);
+ HttpResponse response =
+ httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+
+ if (response.statusCode() != 200 && response.statusCode() != 202) {
+ EndeeApiException.raiseException(response.statusCode(), response.body());
+ }
+
+ @SuppressWarnings("unchecked")
+ Map result = objectMapper.readValue(response.body(), Map.class);
+ return result;
+ } catch (IOException | InterruptedException e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ throw new EndeeException("Failed to rebuild collection", e);
+ }
+ }
+
+ /** Returns the current rebuild status. */
+ public Map rebuildStatus() {
+ try {
+ HttpRequest request = buildGetRequest("/collection/" + name + "/rebuild/status");
+ HttpResponse response =
+ httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+
+ if (response.statusCode() != 200) {
+ EndeeApiException.raiseException(response.statusCode(), response.body());
+ }
+
+ @SuppressWarnings("unchecked")
+ Map result = objectMapper.readValue(response.body(), Map.class);
+ return result;
+ } catch (IOException | InterruptedException e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ throw new EndeeException("Failed to get rebuild status", e);
+ }
+ }
+
+ // ── maintenance ─────────────────────────────────────────────────────────────
+
+ /** Defragments the collection's storage in place. */
+ public Map shrink() {
+ try {
+ HttpRequest request =
+ buildPostJsonRequest("/collection/" + name + "/shrink", "{}");
+ HttpResponse response =
+ httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+
+ if (response.statusCode() != 200) {
+ EndeeApiException.raiseException(response.statusCode(), response.body());
+ }
+
+ @SuppressWarnings("unchecked")
+ Map result = objectMapper.readValue(response.body(), Map.class);
+ return result;
+ } catch (IOException | InterruptedException e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ throw new EndeeException("Failed to shrink collection", e);
+ }
+ }
+
+ /** Creates a backup of this collection (async). */
+ public Map createBackup(String backupName) {
+ if (backupName == null || backupName.isEmpty()) {
+ throw new IllegalArgumentException("backup name is required");
+ }
+
+ try {
+ String jsonBody = JsonUtils.toJson(Map.of("name", backupName));
+ HttpRequest request =
+ buildPostJsonRequest("/collection/" + name + "/backup", jsonBody);
+ HttpResponse response =
+ httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+
+ if (response.statusCode() != 200
+ && response.statusCode() != 201
+ && response.statusCode() != 202) {
+ EndeeApiException.raiseException(response.statusCode(), response.body());
+ }
+
+ @SuppressWarnings("unchecked")
+ Map result = objectMapper.readValue(response.body(), Map.class);
+ return result;
+ } catch (IOException | InterruptedException e) {
+ if (e instanceof InterruptedException) {
+ Thread.currentThread().interrupt();
+ }
+ throw new EndeeException("Failed to create backup", e);
+ }
+ }
+
+ // ── vector normalization helpers ────────────────────────────────────────────
+
+ private static double[] normalizeDense(double[] vector, String spaceType) {
+ if (!"cosine".equals(spaceType)) {
+ return vector;
+ }
+ double norm = computeNorm(vector);
+ if (norm < 1e-10) {
+ return vector;
+ }
+ double[] normalized = new double[vector.length];
+ for (int i = 0; i < vector.length; i++) {
+ normalized[i] = vector[i] / norm;
+ }
+ return normalized;
+ }
+
+ private static double computeNorm(double[] vector) {
+ double sumSquares = 0;
+ for (double v : vector) {
+ sumSquares += v * v;
+ }
+ return Math.sqrt(sumSquares);
+ }
+
+ private static void validateVectorValues(double[] vector, String objectId) {
+ for (double v : vector) {
+ if (Double.isNaN(v) || Double.isInfinite(v)) {
+ throw new IllegalArgumentException(
+ "Object '" + objectId + "' contains non-finite vector value (NaN or Inf)");
+ }
+ }
+ }
+
+ // ── HTTP helpers ────────────────────────────────────────────────────────────
+
+ private HttpRequest buildGetRequest(String path) {
+ HttpRequest.Builder builder =
+ HttpRequest.newBuilder()
+ .uri(URI.create(baseUrl + path))
+ .header("Content-Type", "application/json")
+ .timeout(DEFAULT_TIMEOUT)
+ .GET();
+
+ if (token != null && !token.isBlank()) {
+ builder.header("Authorization", token);
+ }
+ return builder.build();
+ }
+
+ private HttpRequest buildPostJsonRequest(String path, String jsonBody) {
+ HttpRequest.Builder builder =
+ HttpRequest.newBuilder()
+ .uri(URI.create(baseUrl + path))
+ .header("Content-Type", "application/json")
+ .timeout(DEFAULT_TIMEOUT)
+ .POST(HttpRequest.BodyPublishers.ofString(jsonBody));
+
+ if (token != null && !token.isBlank()) {
+ builder.header("Authorization", token);
+ }
+ return builder.build();
+ }
+
+ private HttpRequest buildPostMsgpackRequest(String path, byte[] body) {
+ HttpRequest.Builder builder =
+ HttpRequest.newBuilder()
+ .uri(URI.create(baseUrl + path))
+ .header("Content-Type", "application/msgpack")
+ .timeout(DEFAULT_TIMEOUT)
+ .POST(HttpRequest.BodyPublishers.ofByteArray(body));
+
+ if (token != null && !token.isBlank()) {
+ builder.header("Authorization", token);
+ }
+ return builder.build();
+ }
+
+ private HttpRequest buildDeleteRequest(String path) {
+ HttpRequest.Builder builder =
+ HttpRequest.newBuilder()
+ .uri(URI.create(baseUrl + path))
+ .timeout(DEFAULT_TIMEOUT)
+ .DELETE();
+
+ if (token != null && !token.isBlank()) {
+ builder.header("Authorization", token);
+ }
+ return builder.build();
+ }
+
+ private HttpRequest buildDeleteJsonRequest(String path, String jsonBody) {
+ HttpRequest.Builder builder =
+ HttpRequest.newBuilder()
+ .uri(URI.create(baseUrl + path))
+ .header("Content-Type", "application/json")
+ .timeout(DEFAULT_TIMEOUT)
+ .method("DELETE", HttpRequest.BodyPublishers.ofString(jsonBody));
+
+ if (token != null && !token.isBlank()) {
+ builder.header("Authorization", token);
+ }
+ return builder.build();
+ }
+}
diff --git a/src/main/java/io/endee/client/Endee.java b/src/main/java/io/endee/client/Endee.java
index 9129141..9a78ba6 100644
--- a/src/main/java/io/endee/client/Endee.java
+++ b/src/main/java/io/endee/client/Endee.java
@@ -1,61 +1,55 @@
package io.endee.client;
-import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.endee.client.exception.EndeeApiException;
import io.endee.client.exception.EndeeException;
-import io.endee.client.types.CreateIndexOptions;
-import io.endee.client.types.IndexInfo;
-import io.endee.client.types.Precision;
-import io.endee.client.types.SpaceType;
-import io.endee.client.util.JsonUtils;
import io.endee.client.util.ValidationUtils;
import java.io.IOException;
import java.net.URI;
+import java.net.URLEncoder;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.time.Duration;
import java.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Main Endee client for Endee-DB.
+ * Main Endee client for the Endee vector database (v2 Collections API).
*
* Example usage:
*
*
{@code
- * Endee client = new Endee("auth-token");
+ * Endee client = new Endee("db_name:secret:region");
*
- * // Create an index
- * CreateIndexOptions options = CreateIndexOptions.builder("my_index", 128)
- * .spaceType(SpaceType.COSINE)
- * .precision(Precision.INT8)
- * .build();
- * client.createIndex(options);
+ * // Create a collection
+ * client.createCollection("my_docs", List.of(
+ * Map.of("name", "embedding", "type", "vector",
+ * "params", Map.of("dimension", 768, "space_type", "cosine", "precision", "int8")),
+ * Map.of("name", "keywords", "type", "sparse", "sparse_model", "default")
+ * ));
*
- * // Get an index and perform operations
- * Index index = client.getIndex("my_index");
+ * // Get a collection
+ * Collection collection = client.getCollection("my_docs");
* }
*/
public class Endee {
private static final Logger logger = LoggerFactory.getLogger(Endee.class);
private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(30);
- private static final int MAX_DIMENSION = 8000;
- private static final int MIN_DIMENSION = 2;
- private static final List VALID_SPARSE_MODELS = List.of("default", "endee_bm25");
+ private static final Set VALID_DB_TYPES =
+ Set.of("starter", "pro", "scale", "enterprise");
+ private static final Set VALID_TOKEN_TYPES = Set.of("rw", "r");
private String token;
private String baseUrl;
- private final int version;
private final HttpClient httpClient;
private final ObjectMapper objectMapper;
- /**
- * Creates a new Endee client without authentication. Uses local server at
- * http://127.0.0.1:8080/api/v1
- */
+ /** Creates a new Endee client without authentication. Uses local server. */
public Endee() {
this(null);
}
@@ -63,19 +57,18 @@ public Endee() {
/**
* Creates a new Endee client.
*
- * @param token the Auth token (optional). Format: {@code "account:password"} or {@code
- * "account:password:region"}
+ * @param token the auth token. Format: {@code "db_name:secret"} or {@code
+ * "db_name:secret:region"}
*/
public Endee(String token) {
this.token = token;
- this.baseUrl = "http://127.0.0.1:8080/api/v1";
- this.version = 1;
+ this.baseUrl = "http://127.0.0.1:8080/api/v2";
this.objectMapper = new ObjectMapper();
if (token != null && !token.isEmpty()) {
String[] tokenParts = token.split(":");
if (tokenParts.length > 2) {
- this.baseUrl = "https://" + tokenParts[2] + ".endee.io/api/v1";
+ this.baseUrl = "https://" + tokenParts[2] + ".endee.io/api/v2";
this.token = tokenParts[0] + ":" + tokenParts[1];
}
}
@@ -87,220 +80,455 @@ public Endee(String token) {
.build();
}
- /**
- * Sets a custom base URL for the API.
- *
- * @param url the base URL
- * @return the URL that was set
- */
- public String setBaseUrl(String url) {
+ /** Sets a custom base URL for the API. */
+ public void setBaseUrl(String url) {
this.baseUrl = url;
- return url;
}
+ /** Sets the authentication token. */
+ public void setToken(String token) {
+ this.token = token;
+ }
+
+ // ── Collection API ──────────────────────────────────────────────────────────
+
/**
- * Creates a new index.
+ * Creates a new collection with typed fields.
*
- * @param options the index creation options
- * @return success message
- * @throws EndeeException if the operation fails
+ * @param name collection name
+ * @param fields list of field definitions as maps
+ * @return server response
*/
- public String createIndex(CreateIndexOptions options) {
- if (!ValidationUtils.isValidIndexName(options.getName())) {
+ public Map createCollection(String name, List> fields) {
+ if (!ValidationUtils.isValidCollectionName(name)) {
throw new IllegalArgumentException(
- "Invalid index name. Must be alphanumeric with underscores, max 48 characters.");
+ "Invalid collection name. Must be alphanumeric with underscores, max 48 chars, no '__' prefix.");
}
- if (options.getDimension() < MIN_DIMENSION || options.getDimension() > MAX_DIMENSION) {
- throw new IllegalArgumentException(
- "Dimension must be between " + MIN_DIMENSION + " and " + MAX_DIMENSION);
+ if (fields == null || fields.isEmpty()) {
+ throw new IllegalArgumentException("At least one field is required");
}
- String normalizedSpaceType = options.getSpaceType().getValue().toLowerCase();
- if (!List.of("cosine", "l2", "ip").contains(normalizedSpaceType)) {
- throw new IllegalArgumentException("Invalid space type: " + options.getSpaceType());
- }
+ Map data = new LinkedHashMap<>();
+ data.put("name", name);
+ data.put("fields", fields);
- String sparseModel = options.getSparseModel();
- if (sparseModel != null) {
- String normalized = sparseModel.toLowerCase();
- if (!VALID_SPARSE_MODELS.contains(normalized)) {
- throw new IllegalArgumentException(
- "Invalid sparseModel. Must be one of: " + VALID_SPARSE_MODELS);
- }
- sparseModel = normalized;
- }
+ return call("POST", "/collection", data, Set.of(200, 201));
+ }
- Map data = new HashMap<>();
- data.put("index_name", options.getName());
- data.put("dim", options.getDimension());
- data.put("space_type", normalizedSpaceType);
- data.put("M", options.getM());
- data.put("ef_con", options.getEfCon());
- data.put("checksum", -1);
- data.put("precision", options.getPrecision().getValue());
-
- if (sparseModel != null) {
- data.put("sparse_model", sparseModel);
- }
- if (options.getVersion() != null) {
- data.put("version", options.getVersion());
- }
+ /** Lists all collections. */
+ @SuppressWarnings("unchecked")
+ public List> listCollections() {
+ Map result = call("GET", "/collection", null, Set.of(200));
+ Object collections = result.get("collections");
+ return collections instanceof List ? (List>) collections : List.of();
+ }
- try {
- HttpRequest request = buildPostRequest("/index/create", data);
- HttpResponse response =
- httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+ /**
+ * Gets a Collection object for performing operations.
+ *
+ * @param name collection name
+ * @return Collection object
+ */
+ public Collection getCollection(String name) {
+ Map metadata = call("GET", "/collection/" + name, null, Set.of(200));
+ return new Collection(name, token, baseUrl, metadata);
+ }
- if (response.statusCode() != 200) {
- logger.error("Error: {}", response.body());
- EndeeApiException.raiseException(response.statusCode(), response.body());
- }
+ /** Deletes a collection and all its data. */
+ public Map deleteCollection(String name) {
+ return call("DELETE", "/collection/" + name, null, Set.of(200));
+ }
- return "Index created successfully";
- } catch (IOException | InterruptedException e) {
- if (e instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- throw new EndeeException("Failed to create index", e);
- }
+ // ── Database Admin (root token) ─────────────────────────────────────────────
+
+ /** Creates a database. Returns the new db token string. */
+ public String createDatabase(String dbName, String dbType) {
+ requireNonEmpty(dbName, "db_name");
+ String dt = dbType != null ? dbType.toLowerCase() : "enterprise";
+ validateIn(dt, VALID_DB_TYPES, "db_type");
+ Map result =
+ call("POST", "/admin/dbs", Map.of("db_name", dbName, "db_type", dt), Set.of(200, 201));
+ return (String) result.get("db_token");
+ }
+
+ /** Creates a database with default type "enterprise". */
+ public String createDatabase(String dbName) {
+ return createDatabase(dbName, "enterprise");
+ }
+
+ /** Lists all databases. */
+ @SuppressWarnings("unchecked")
+ public List> listDatabases() {
+ Map result = call("GET", "/admin/dbs", null, Set.of(200));
+ Object dbs = result.get("dbs");
+ return dbs instanceof List ? (List>) dbs : List.of();
+ }
+
+ /** Gets a single database's info. */
+ public Map getDatabase(String dbName) {
+ requireNonEmpty(dbName, "db_name");
+ return call("GET", "/dbs/" + dbName + "/info", null, Set.of(200));
+ }
+
+ /** Deletes a database and all its data. */
+ public Map deleteDatabase(String dbName) {
+ requireNonEmpty(dbName, "db_name");
+ return call("DELETE", "/admin/dbs/" + dbName, null, Set.of(200));
+ }
+
+ /** Activates a previously deactivated database. */
+ public Map activateDatabase(String dbName) {
+ return call("POST", "/admin/dbs/" + dbName + "/activate", null, Set.of(200));
+ }
+
+ /** Deactivates a database. */
+ public Map deactivateDatabase(String dbName) {
+ return call("POST", "/admin/dbs/" + dbName + "/deactivate", null, Set.of(200));
+ }
+
+ /** Changes a database's tier. */
+ public Map setDatabaseType(String dbName, String dbType) {
+ String dt = dbType.toLowerCase();
+ validateIn(dt, VALID_DB_TYPES, "db_type");
+ return call("PUT", "/admin/dbs/" + dbName + "/type", Map.of("db_type", dt), Set.of(200));
+ }
+
+ // ── Admin collection views ──────────────────────────────────────────────────
+
+ /** Lists collections in a specific database. */
+ @SuppressWarnings("unchecked")
+ public List> listDbCollections(String dbName) {
+ requireNonEmpty(dbName, "db_name");
+ Map result =
+ call("GET", "/admin/dbs/" + dbName + "/collection", null, Set.of(200));
+ Object c = result.get("collections");
+ return c instanceof List ? (List>) c : List.of();
+ }
+
+ /** Lists all collections across all databases. */
+ @SuppressWarnings("unchecked")
+ public List> listAllCollections() {
+ Map result = call("GET", "/admin/collection", null, Set.of(200));
+ Object c = result.get("collections");
+ return c instanceof List ? (List>) c : List.of();
+ }
+
+ /** Deletes a collection inside a specific database. */
+ public Map deleteDbCollection(String dbName, String collectionName) {
+ requireNonEmpty(dbName, "db_name");
+ requireNonEmpty(collectionName, "collection_name");
+ return call(
+ "DELETE", "/admin/dbs/" + dbName + "/collection/" + collectionName, null, Set.of(200));
+ }
+
+ // ── Token management (admin) ────────────────────────────────────────────────
+
+ /** Creates a token for a database. Returns the new db token string. */
+ public String createToken(String dbName, String name, String tokenType) {
+ requireNonEmpty(dbName, "db_name");
+ requireNonEmpty(name, "name");
+ String tt = tokenType != null ? tokenType.toLowerCase() : "rw";
+ validateIn(tt, VALID_TOKEN_TYPES, "token_type");
+ Map result =
+ call(
+ "POST",
+ "/admin/dbs/" + dbName + "/tokens",
+ Map.of("name", name, "token_type", tt),
+ Set.of(200, 201));
+ return (String) result.get("db_token");
+ }
+
+ /** Creates a read-write token. */
+ public String createToken(String dbName, String name) {
+ return createToken(dbName, name, "rw");
+ }
+
+ /** Lists a database's tokens. */
+ @SuppressWarnings("unchecked")
+ public List> listTokens(String dbName) {
+ requireNonEmpty(dbName, "db_name");
+ Map result =
+ call("GET", "/admin/dbs/" + dbName + "/tokens", null, Set.of(200));
+ Object t = result.get("tokens");
+ return t instanceof List ? (List>) t : List.of();
+ }
+
+ /** Deletes a database token by name. */
+ public Map deleteToken(String dbName, String name) {
+ requireNonEmpty(dbName, "db_name");
+ requireNonEmpty(name, "name");
+ return call("DELETE", "/admin/dbs/" + dbName + "/tokens/" + name, null, Set.of(200));
+ }
+
+ // ── Self-service token management ───────────────────────────────────────────
+
+ /** Creates a token for your own database. Returns the new db token string. */
+ public String createMyToken(String name, String tokenType) {
+ requireNonEmpty(name, "name");
+ String tt = tokenType != null ? tokenType.toLowerCase() : "rw";
+ validateIn(tt, VALID_TOKEN_TYPES, "token_type");
+ Map result =
+ call("POST", "/tokens", Map.of("name", name, "token_type", tt), Set.of(200, 201));
+ return (String) result.get("db_token");
+ }
+
+ /** Creates a read-write token for your own database. */
+ public String createMyToken(String name) {
+ return createMyToken(name, "rw");
+ }
+
+ /** Lists your own database's tokens. */
+ @SuppressWarnings("unchecked")
+ public List> listMyTokens() {
+ Map result = call("GET", "/tokens", null, Set.of(200));
+ Object t = result.get("tokens");
+ return t instanceof List ? (List>) t : List.of();
+ }
+
+ /** Deletes one of your own database's tokens by name. */
+ public Map deleteMyToken(String name) {
+ requireNonEmpty(name, "name");
+ return call("DELETE", "/tokens/" + name, null, Set.of(200));
+ }
+
+ // ── Server info ─────────────────────────────────────────────────────────────
+
+ /** Server health check. Returns {status, timestamp}. */
+ public Map health() {
+ return call("GET", "/health", null, Set.of(200));
+ }
+
+ /** Server stats. Returns {version, uptime, total_requests}. */
+ public Map stats() {
+ return call("GET", "/stats", null, Set.of(200));
+ }
+
+ // ── Backups ─────────────────────────────────────────────────────────────────
+
+ /** Lists this database's backups. */
+ public Object listBackups() {
+ return call("GET", "/backup", null, Set.of(200));
+ }
+
+ /** Gets metadata for one backup. */
+ public Map backupInfo(String backupName) {
+ requireNonEmpty(backupName, "backup_name");
+ return call("GET", "/backup/" + backupName + "/info", null, Set.of(200));
+ }
+
+ /** Gets the in-progress backup status. */
+ public Map activeBackup() {
+ return call("GET", "/backup/active", null, Set.of(200));
+ }
+
+ /** Restores a backup into a new collection. */
+ public Map restoreBackup(String backupName, String targetCollectionName) {
+ requireNonEmpty(backupName, "backup_name");
+ requireNonEmpty(targetCollectionName, "target_collection_name");
+ return call(
+ "POST",
+ "/backup/" + backupName + "/restore",
+ Map.of("target_collection_name", targetCollectionName),
+ Set.of(200, 201));
+ }
+
+ /** Deletes a backup. */
+ public Map deleteBackup(String backupName) {
+ requireNonEmpty(backupName, "backup_name");
+ return call("DELETE", "/backup/" + backupName, null, Set.of(200, 204));
}
/**
- * Lists all indexes.
+ * Downloads a backup as a .tar file.
*
- * @return raw JSON string of index information
- * @throws EndeeException if the operation fails
+ * @param backupName name of the backup
+ * @param destPath local file path to write the .tar to
+ * @param dbName optional database name (for root-token multi-db targeting)
+ * @return the destination path
*/
- public String listIndexes() {
- try {
- HttpRequest request = buildGetRequest("/index/list");
- HttpResponse response =
- httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+ public String downloadBackup(String backupName, String destPath, String dbName) {
+ requireNonEmpty(backupName, "backup_name");
+ requireNonEmpty(destPath, "dest_path");
+
+ StringBuilder url = new StringBuilder(baseUrl)
+ .append("/backup/")
+ .append(backupName)
+ .append("/download?token=")
+ .append(URLEncoder.encode(token != null ? token : "", StandardCharsets.UTF_8));
+ if (dbName != null && !dbName.isEmpty()) {
+ url.append("&db=").append(URLEncoder.encode(dbName, StandardCharsets.UTF_8));
+ }
- return response.body();
- } catch (IOException | InterruptedException e) {
- if (e instanceof InterruptedException) {
- Thread.currentThread().interrupt();
+ try {
+ HttpRequest request = HttpRequest.newBuilder()
+ .uri(URI.create(url.toString()))
+ .timeout(DEFAULT_TIMEOUT)
+ .GET()
+ .build();
+ HttpResponse response =
+ httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray());
+ if (response.statusCode() != 200) {
+ EndeeApiException.raiseException(response.statusCode(), new String(response.body()));
}
- throw new EndeeException("Failed to list indexes", e);
+ Files.write(Path.of(destPath), response.body());
+ return destPath;
+ } catch (EndeeException e) {
+ throw e;
+ } catch (IOException | InterruptedException e) {
+ if (e instanceof InterruptedException) Thread.currentThread().interrupt();
+ throw new EndeeException("Download backup failed", e);
}
}
+ /** Downloads a backup (no db_name). */
+ public String downloadBackup(String backupName, String destPath) {
+ return downloadBackup(backupName, destPath, null);
+ }
+
/**
- * Deletes an index.
+ * Uploads a backup .tar file via multipart.
*
- * @param name the index name to delete
- * @return success message
- * @throws EndeeException if the operation fails
+ * @param filePath path to a .tar backup file
+ * @return server response
*/
- public String deleteIndex(String name) {
+ @SuppressWarnings("unchecked")
+ public Map uploadBackup(String filePath) {
+ requireNonEmpty(filePath, "file_path");
+ Path path = Path.of(filePath);
+ String fileName = path.getFileName().toString();
+ if (!fileName.endsWith(".tar")) {
+ throw new IllegalArgumentException("backup file must be a .tar");
+ }
+
try {
- HttpRequest request = buildDeleteRequest("/index/" + name + "/delete");
+ byte[] fileBytes = Files.readAllBytes(path);
+ String boundary = "----EndeeBackupBoundary" + System.nanoTime();
+
+ byte[] multipartBody = buildMultipartBody(boundary, "backup", fileName, fileBytes);
+
+ HttpRequest.Builder builder = HttpRequest.newBuilder()
+ .uri(URI.create(baseUrl + "/backup/upload"))
+ .timeout(DEFAULT_TIMEOUT)
+ .header("Content-Type", "multipart/form-data; boundary=" + boundary)
+ .POST(HttpRequest.BodyPublishers.ofByteArray(multipartBody));
+
+ if (token != null && !token.isEmpty()) {
+ builder.header("Authorization", token);
+ }
+
HttpResponse response =
- httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+ httpClient.send(builder.build(), HttpResponse.BodyHandlers.ofString());
- if (response.statusCode() != 200) {
- logger.error("Error: {}", response.body());
+ if (response.statusCode() != 200 && response.statusCode() != 201) {
EndeeApiException.raiseException(response.statusCode(), response.body());
}
- return "Index " + name + " deleted successfully";
- } catch (IOException | InterruptedException e) {
- if (e instanceof InterruptedException) {
- Thread.currentThread().interrupt();
+ String body = response.body();
+ if (body == null || body.isBlank()) return Map.of();
+ try {
+ return objectMapper.readValue(body, Map.class);
+ } catch (Exception e) {
+ return Map.of("message", body);
}
- throw new EndeeException("Failed to delete index", e);
+ } catch (EndeeException e) {
+ throw e;
+ } catch (IOException | InterruptedException e) {
+ if (e instanceof InterruptedException) Thread.currentThread().interrupt();
+ throw new EndeeException("Upload backup failed", e);
}
}
- /**
- * Gets an index by name.
- *
- * @param name the index name
- * @return the Index object for performing vector operations
- * @throws EndeeException if the operation fails
- */
- public Index getIndex(String name) {
+ private static byte[] buildMultipartBody(
+ String boundary, String fieldName, String fileName, byte[] fileBytes) throws IOException {
+ String CRLF = "\r\n";
+ var baos = new java.io.ByteArrayOutputStream();
+ baos.write(("--" + boundary + CRLF).getBytes(StandardCharsets.UTF_8));
+ baos.write(("Content-Disposition: form-data; name=\"" + fieldName + "\"; filename=\""
+ + fileName + "\"" + CRLF).getBytes(StandardCharsets.UTF_8));
+ baos.write(("Content-Type: application/x-tar" + CRLF).getBytes(StandardCharsets.UTF_8));
+ baos.write(CRLF.getBytes(StandardCharsets.UTF_8));
+ baos.write(fileBytes);
+ baos.write((CRLF + "--" + boundary + "--" + CRLF).getBytes(StandardCharsets.UTF_8));
+ return baos.toByteArray();
+ }
+
+ // ── Internal HTTP helpers ───────────────────────────────────────────────────
+
+ @SuppressWarnings("unchecked")
+ private Map call(
+ String method, String path, Map json, Set okStatuses) {
try {
- HttpRequest request = buildGetRequest("/index/" + name + "/info");
+ HttpRequest request = buildRequest(method, path, json);
HttpResponse response =
httpClient.send(request, HttpResponse.BodyHandlers.ofString());
- if (response.statusCode() != 200) {
+ if (!okStatuses.contains(response.statusCode())) {
+ logger.error("Error: {}", response.body());
EndeeApiException.raiseException(response.statusCode(), response.body());
}
- JsonNode data = objectMapper.readTree(response.body());
-
- IndexInfo indexInfo = new IndexInfo();
- indexInfo.setSpaceType(SpaceType.fromValue(data.get("space_type").asText()));
- indexInfo.setDimension(data.get("dimension").asInt());
- indexInfo.setTotalElements(data.get("total_elements").asLong());
- indexInfo.setPrecision(Precision.fromValue(data.get("precision").asText()));
- indexInfo.setM(data.get("M").asInt());
- indexInfo.setEfCon(data.get("ef_con").asInt());
-
- if (data.has("checksum") && !data.get("checksum").isNull()) {
- indexInfo.setChecksum(data.get("checksum").asLong());
+ String body = response.body();
+ if (body == null || body.isBlank()) {
+ return Map.of();
}
- if (data.has("version") && !data.get("version").isNull()) {
- indexInfo.setVersion(data.get("version").asInt());
- }
- if (data.has("sparse_model") && !data.get("sparse_model").isNull()) {
- indexInfo.setSparseModel(data.get("sparse_model").asText());
- }
- if (data.has("lib_token") && !data.get("lib_token").isNull()) {
- indexInfo.setLibToken(data.get("lib_token").asText());
- }
-
- return new Index(name, token, baseUrl, version, indexInfo);
+ return objectMapper.readValue(body, Map.class);
+ } catch (EndeeException e) {
+ throw e;
} catch (IOException | InterruptedException e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt();
}
- throw new EndeeException("Failed to get index", e);
+ throw new EndeeException("Request failed: " + method + " " + path, e);
}
}
- private HttpRequest buildGetRequest(String path) {
+ private HttpRequest buildRequest(String method, String path, Map json) {
HttpRequest.Builder builder =
HttpRequest.newBuilder()
.uri(URI.create(baseUrl + path))
- .header("Content-Type", "application/json")
- .timeout(DEFAULT_TIMEOUT)
- .GET();
+ .timeout(DEFAULT_TIMEOUT);
if (token != null && !token.isEmpty()) {
builder.header("Authorization", token);
}
- return builder.build();
- }
-
- private HttpRequest buildPostRequest(String path, Map data) {
- String json = JsonUtils.toJson(data);
- HttpRequest.Builder builder =
- HttpRequest.newBuilder()
- .uri(URI.create(baseUrl + path))
- .header("Content-Type", "application/json")
- .timeout(DEFAULT_TIMEOUT)
- .POST(HttpRequest.BodyPublishers.ofString(json));
-
- if (token != null && !token.isEmpty()) {
- builder.header("Authorization", token);
+ if (json != null) {
+ String jsonBody;
+ try {
+ jsonBody = objectMapper.writeValueAsString(json);
+ } catch (Exception e) {
+ throw new EndeeException("Failed to serialize request body", e);
+ }
+ builder.header("Content-Type", "application/json");
+ builder.method(method, HttpRequest.BodyPublishers.ofString(jsonBody));
+ } else {
+ switch (method) {
+ case "GET" -> builder.GET();
+ case "DELETE" -> builder.DELETE();
+ case "POST" ->
+ builder
+ .header("Content-Type", "application/json")
+ .POST(HttpRequest.BodyPublishers.noBody());
+ case "PUT" ->
+ builder
+ .header("Content-Type", "application/json")
+ .PUT(HttpRequest.BodyPublishers.noBody());
+ default -> builder.method(method, HttpRequest.BodyPublishers.noBody());
+ }
}
return builder.build();
}
- private HttpRequest buildDeleteRequest(String path) {
- HttpRequest.Builder builder =
- HttpRequest.newBuilder().uri(URI.create(baseUrl + path)).timeout(DEFAULT_TIMEOUT).DELETE();
-
- if (token != null && !token.isEmpty()) {
- builder.header("Authorization", token);
+ private static void requireNonEmpty(String value, String name) {
+ if (value == null || value.isEmpty()) {
+ throw new IllegalArgumentException(name + " is required");
}
+ }
- return builder.build();
+ private static void validateIn(String value, Set valid, String name) {
+ if (!valid.contains(value)) {
+ throw new IllegalArgumentException(name + " must be one of " + valid);
+ }
}
}
diff --git a/src/main/java/io/endee/client/Index.java b/src/main/java/io/endee/client/Index.java
deleted file mode 100644
index e74c7fd..0000000
--- a/src/main/java/io/endee/client/Index.java
+++ /dev/null
@@ -1,711 +0,0 @@
-package io.endee.client;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import io.endee.client.exception.EndeeApiException;
-import io.endee.client.exception.EndeeException;
-import io.endee.client.types.*;
-import io.endee.client.util.CryptoUtils;
-import io.endee.client.util.JsonUtils;
-import io.endee.client.util.MessagePackUtils;
-import io.endee.client.util.ValidationUtils;
-import java.io.IOException;
-import java.net.URI;
-import java.net.http.HttpClient;
-import java.net.http.HttpRequest;
-import java.net.http.HttpResponse;
-import java.time.Duration;
-import java.util.*;
-import java.util.stream.Collectors;
-
-/**
- * Index client for Endee-DB vector operations.
- *
- * Example usage:
- *
- *
{@code
- * Index index = client.getIndex("my_index");
- *
- * // Upsert vectors
- * List vectors = List.of(
- * VectorItem.builder("vec1", new double[] { 0.1, 0.2, 0.3 })
- * .meta(Map.of("label", "example"))
- * .build());
- * index.upsert(vectors);
- *
- * // Query
- * List results = index.query(
- * QueryOptions.builder()
- * .vector(new double[] { 0.1, 0.2, 0.3 })
- * .topK(10)
- * .build());
- * }
- */
-public class Index {
- private static final Duration DEFAULT_TIMEOUT = Duration.ofSeconds(30);
- private static final int MAX_BATCH_SIZE = 1000;
- private static final int MAX_TOP_K = 4096;
- private static final int MAX_EF = 1024;
- private static final int MAX_FILTER_BOOST_PERCENTAGE = 400;
-
- private final String name;
- private final String token;
- private final String url;
- private final HttpClient httpClient;
- private final ObjectMapper objectMapper;
-
- private long count;
- private SpaceType spaceType;
- private int dimension;
- private Precision precision;
- private int m;
- private String sparseModel;
- private String libToken;
- private int efCon;
-
- /** Creates a new Index instance. */
- public Index(String name, String token, String url, int version, IndexInfo params) {
- this.name = name;
- this.token = token;
- this.url = url;
- this.objectMapper = new ObjectMapper();
-
- this.count = params != null ? params.getTotalElements() : 0;
- this.spaceType =
- params != null && params.getSpaceType() != null ? params.getSpaceType() : SpaceType.COSINE;
- this.dimension = params != null ? params.getDimension() : 0;
- this.precision =
- params != null && params.getPrecision() != null ? params.getPrecision() : Precision.INT8;
- this.m = params != null ? params.getM() : 16;
- this.sparseModel = params != null ? params.getSparseModel() : "None";
- this.libToken = params != null ? params.getLibToken() : null;
- this.efCon = params != null ? params.getEfCon() : 128;
-
- this.httpClient =
- HttpClient.newBuilder()
- .version(HttpClient.Version.HTTP_2)
- .connectTimeout(DEFAULT_TIMEOUT)
- .build();
- }
-
- @Override
- public String toString() {
- return name;
- }
-
- public String getLibToken() {
- return libToken;
- }
-
- /**
- * Returns {@code true} when this index supports hybrid (sparse + dense) vectors. Determined by
- * {@code sparse_model != "None"} from the server response.
- */
- public boolean isHybrid() {
- return sparseModel != null && !"None".equals(sparseModel);
- }
-
- /** Normalizes a vector for cosine similarity. Returns [normalizedVector, norm]. */
- private double[][] normalizeVector(double[] vector) {
- if (vector.length != dimension) {
- throw new IllegalArgumentException(
- "Vector dimension mismatch: expected " + dimension + ", got " + vector.length);
- }
-
- if (spaceType != SpaceType.COSINE) {
- return new double[][] {vector, {1.0}};
- }
-
- double sumSquares = 0;
- for (double v : vector) {
- sumSquares += v * v;
- }
- double norm = Math.sqrt(sumSquares);
-
- if (norm == 0) {
- return new double[][] {vector, {1.0}};
- }
-
- double[] normalized = new double[vector.length];
- for (int i = 0; i < vector.length; i++) {
- normalized[i] = vector[i] / norm;
- }
-
- return new double[][] {normalized, {norm}};
- }
-
- /** Validates that a vector contains only finite values (no NaN or Inf). */
- private static void validateVectorValues(double[] vector, String vectorId) {
- for (double v : vector) {
- if (Double.isNaN(v) || Double.isInfinite(v)) {
- throw new IllegalArgumentException(
- "Vector '" + vectorId + "' contains non-finite value (NaN or Inf)");
- }
- }
- }
-
- /**
- * Upserts vectors into the index.
- *
- * @param inputArray list of vector items to upsert (1 – 1,000 items)
- * @return success message
- */
- public String upsert(List inputArray) {
- if (inputArray.isEmpty()) {
- throw new IllegalArgumentException("Must provide at least one vector to upsert");
- }
- if (inputArray.size() > MAX_BATCH_SIZE) {
- throw new IllegalArgumentException(
- "Cannot insert more than " + MAX_BATCH_SIZE + " vectors at a time");
- }
-
- List ids =
- inputArray.stream()
- .map(item -> item.getId() != null ? item.getId() : "")
- .collect(Collectors.toList());
- ValidationUtils.validateVectorIds(ids);
-
- List vectorBatch = new ArrayList<>();
-
- for (VectorItem item : inputArray) {
- validateVectorValues(item.getVector(), item.getId());
- double[][] result = normalizeVector(item.getVector());
- double[] normalizedVector = result[0];
- double norm = result[1][0];
-
- byte[] metaData = CryptoUtils.jsonZip(item.getMeta() != null ? item.getMeta() : Map.of());
-
- int[] sparseIndices = item.getSparseIndices() != null ? item.getSparseIndices() : new int[0];
- double[] sparseValues =
- item.getSparseValues() != null ? item.getSparseValues() : new double[0];
-
- if (!isHybrid() && (sparseIndices.length > 0 || sparseValues.length > 0)) {
- throw new IllegalArgumentException(
- "Cannot insert sparse data into a dense-only index. Use sparseModel(\"default\") when creating the index.");
- }
-
- if (isHybrid()) {
- if (sparseIndices.length == 0 || sparseValues.length == 0) {
- throw new IllegalArgumentException(
- "Both sparse_indices and sparse_values must be provided for hybrid vectors.");
- }
- if (sparseIndices.length != sparseValues.length) {
- throw new IllegalArgumentException(
- "sparseIndices and sparseValues must have the same length. Got "
- + sparseIndices.length
- + " indices and "
- + sparseValues.length
- + " values.");
- }
- }
-
- String filterJson = JsonUtils.toJson(item.getFilter() != null ? item.getFilter() : Map.of());
-
- if (isHybrid()) {
- vectorBatch.add(
- new Object[] {
- item.getId(),
- metaData,
- filterJson,
- norm,
- normalizedVector,
- sparseIndices,
- sparseValues
- });
- } else {
- vectorBatch.add(new Object[] {item.getId(), metaData, filterJson, norm, normalizedVector});
- }
- }
-
- byte[] serializedData = MessagePackUtils.packVectors(vectorBatch);
-
- try {
- HttpRequest request =
- buildPostMsgpackRequest("/index/" + name + "/vector/insert", serializedData);
- HttpResponse response =
- httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray());
-
- if (response.statusCode() != 200) {
- EndeeApiException.raiseException(response.statusCode(), new String(response.body()));
- }
-
- return "Vectors inserted successfully";
- } catch (IOException | InterruptedException e) {
- if (e instanceof InterruptedException) {
- Thread.currentThread().interrupt();
- }
- throw new EndeeException("Failed to upsert vectors", e);
- }
- }
-
- /**
- * Queries the index for similar vectors.
- *
- * @param options the query options
- * @return list of query results
- */
- public List query(QueryOptions options) {
- if (options.getTopK() < 1 || options.getTopK() > MAX_TOP_K) {
- throw new IllegalArgumentException("top_k must be between 1 and " + MAX_TOP_K);
- }
- if (options.getEf() > MAX_EF) {
- throw new IllegalArgumentException("ef cannot be greater than " + MAX_EF);
- }
- if (options.getPrefilterCardinalityThreshold() < 1_000
- || options.getPrefilterCardinalityThreshold() > 1_000_000) {
- throw new IllegalArgumentException(
- "prefilterCardinalityThreshold must be between 1,000 and 1,000,000");
- }
- if (options.getFilterBoostPercentage() < 0
- || options.getFilterBoostPercentage() > MAX_FILTER_BOOST_PERCENTAGE) {
- throw new IllegalArgumentException(
- "filterBoostPercentage must be between 0 and " + MAX_FILTER_BOOST_PERCENTAGE);
- }
- if (options.getDenseRrfWeight() < 0.0 || options.getDenseRrfWeight() > 1.0) {
- throw new IllegalArgumentException("denseRrfWeight must be between 0.0 and 1.0");
- }
- if (options.getRrfRankConstant() < 1) {
- throw new IllegalArgumentException("rrfRankConstant must be at least 1");
- }
-
- boolean hasSparse =
- options.getSparseIndices() != null
- && options.getSparseIndices().length > 0
- && options.getSparseValues() != null
- && options.getSparseValues().length > 0;
- boolean hasDense = options.getVector() != null;
-
- if (!hasDense && !hasSparse) {
- throw new IllegalArgumentException(
- "At least one of 'vector' or 'sparseIndices'/'sparseValues' must be provided.");
- }
-
- if (hasSparse && !isHybrid()) {
- throw new IllegalArgumentException("Cannot perform sparse search on a dense-only index.");
- }
-
- if (hasSparse && options.getSparseIndices().length != options.getSparseValues().length) {
- throw new IllegalArgumentException(
- "sparseIndices and sparseValues must have the same length.");
- }
-
- Map data = new HashMap<>();
- data.put("k", options.getTopK());
- data.put("ef", options.getEf());
- data.put("include_vectors", options.isIncludeVectors());
-
- if (hasDense) {
- double[][] result = normalizeVector(options.getVector());
- data.put("vector", result[0]);
- }
-
- if (hasSparse) {
- data.put("sparse_indices", options.getSparseIndices());
- data.put("sparse_values", options.getSparseValues());
- }
-
- if (options.getFilter() != null) {
- data.put("filter", JsonUtils.toJson(options.getFilter()));
- }
-
- Map