Working with Vector Embeddings

-- Create vertex type and property
CREATE VERTEX TYPE Document;
CREATE PROPERTY Document.embedding LIST OF FLOAT;

-- Create vector index with 384 dimensions using COSINE similarity and INT8 quantization
CREATE INDEX ON Document (embedding) LSM_VECTOR METADATA {
  "dimensions": 384,
  "similarity": "COSINE",
  "quantization": "INT8"
};

CREATE INDEX ON Document (embedding) LSM_VECTOR METADATA {
  "dimensions": 768,
  "similarity": "COSINE"
};

CREATE INDEX ON Image (featureVector) LSM_VECTOR METADATA {
  "dimensions": 512,
  "similarity": "DOT_PRODUCT"
};

CREATE INDEX ON Product (attributes) LSM_VECTOR METADATA {
  "dimensions": 256,
  "similarity": "EUCLIDEAN"
};

CREATE INDEX ON VectorVertex (embedding) LSM_VECTOR METADATA {
  "dimensions": 1024,
  "similarity": "COSINE",
  "quantization": "INT8",            -- 2.5x faster search, 4x less memory (recommended)
  "maxConnections": 32,              -- Higher values improve recall but increase memory
  "beamWidth": 200,                  -- Higher values improve accuracy but reduce speed
  "neighborOverflowFactor": 1.2,     -- Candidate neighbor pool multiplier (default: 1.2)
  "alphaDiversityRelaxation": 1.2,   -- Diversity vs distance trade-off (default: 1.2)
  "addHierarchy": true               -- Enable HNSW hierarchical layers (default: false)
};

LSMVectorIndex vectorIndex = (LSMVectorIndex) database.getSchema().getIndexByName("MyType[embedding]");
vectorIndex.buildVectorGraphNow();

-- Recommended for large, complex datasets
CREATE INDEX ON ComplexDoc (embedding) LSM_VECTOR METADATA {
  "dimensions": 1536,
  "similarity": "COSINE",
  "maxConnections": 16,
  "beamWidth": 100,
  "addHierarchy": true  -- Enable for robustness
};

-- Default for most use cases
CREATE INDEX ON StandardDoc (embedding) LSM_VECTOR METADATA {
  "dimensions": 768,
  "similarity": "COSINE",
  "maxConnections": 16,
  "beamWidth": 100
  -- addHierarchy defaults to false
};

-- Optimized large-scale configuration
CREATE INDEX ON LargeDoc (embedding) LSM_VECTOR METADATA {
  "dimensions": 1536,
  "similarity": "COSINE",
  "quantization": "INT8",            -- 4x memory reduction
  "maxConnections": 16,
  "beamWidth": 100,
  "addHierarchy": true,              -- Robustness for large scale
  "storeVectorsInGraph": true,       -- Fast co-located access
  "locationCacheSize": 200000,
  "mutationsBeforeRebuild": 1000
};

CREATE INDEX ON Document (embedding) LSM_VECTOR METADATA {
  "dimensions": 768,
  "similarity": "COSINE",
  "maxConnections": 16,
  "beamWidth": 100,
  "neighborOverflowFactor": 1.2,
  "alphaDiversityRelaxation": 1.2
};

CREATE INDEX ON Document (embedding) LSM_VECTOR METADATA {
  "dimensions": 768,
  "similarity": "COSINE",
  "maxConnections": 32,
  "beamWidth": 200,
  "neighborOverflowFactor": 1.4,
  "alphaDiversityRelaxation": 1.3,
  "addHierarchy": true
};

CREATE INDEX ON Document (embedding) LSM_VECTOR METADATA {
  "dimensions": 768,
  "similarity": "COSINE",
  "maxConnections": 12,
  "beamWidth": 80,
  "neighborOverflowFactor": 1.1,
  "alphaDiversityRelaxation": 1.1
};

CREATE INDEX ON Document (embedding) LSM_VECTOR METADATA {
  "dimensions": 768,
  "similarity": "COSINE",
  "maxConnections": 8,
  "beamWidth": 100,
  "neighborOverflowFactor": 1.2,
  "alphaDiversityRelaxation": 1.2
};

Map<String, Long> stats = lsmIndex.getStats();
System.out.println("Fetched from graph: " + stats.get("vectorFetchFromGraph"));
System.out.println("Fetched from docs: " + stats.get("vectorFetchFromDocuments"));

-- Configure globally for all vector indexes
ALTER DATABASE `arcadedb.vectorIndex.locationCacheSize` 50000;
ALTER DATABASE `arcadedb.vectorIndex.graphBuildCacheSize` 5000;
ALTER DATABASE `arcadedb.vectorIndex.storeVectorsInGraph` false;

-- Or configure per-index for small datasets
CREATE INDEX ON Document (embedding) LSM_VECTOR METADATA {
  "dimensions": 768,
  "similarity": "COSINE",
  "maxConnections": 8,
  "beamWidth": 100,
  "locationCacheSize": 50000,        -- Limit location cache: ~2.8 MB
  "graphBuildCacheSize": 5000,       -- Limit graph building: ~15 MB peak
  "mutationsBeforeRebuild": 150,     -- Moderate rebuild frequency
  "storeVectorsInGraph": false       -- Use document cache for small datasets
};

CREATE INDEX ON Document (embedding) LSM_VECTOR METADATA {
  "dimensions": 768,
  "similarity": "COSINE",
  "maxConnections": 16,
  "beamWidth": 100,
  "locationCacheSize": 100000,       -- Balance memory vs performance
  "graphBuildCacheSize": 10000,      -- Default (30 MB for 768-dim)
  "mutationsBeforeRebuild": 100,     -- Standard rebuild frequency
  "storeVectorsInGraph": false       -- Use document cache for moderate scale
};

-- Optimal for large indexes: combine quantization + graph storage
CREATE INDEX ON Document (embedding) LSM_VECTOR METADATA {
  "dimensions": 1536,
  "similarity": "COSINE",
  "quantization": "INT8",            -- 4x memory reduction
  "maxConnections": 16,
  "beamWidth": 100,
  "locationCacheSize": 200000,       -- Larger cache for hot vectors
  "graphBuildCacheSize": 10000,      -- Standard build cache
  "mutationsBeforeRebuild": 1000,    -- Reduce rebuild frequency (vectors in graph)
  "storeVectorsInGraph": true        -- Co-locate vectors with graph topology
};

CREATE INDEX ON Document (embedding) LSM_VECTOR METADATA {
  "dimensions": 768,
  "similarity": "COSINE",
  "maxConnections": 32,
  "beamWidth": 200,
  "locationCacheSize": -1,           -- Unlimited cache (no eviction)
  "graphBuildCacheSize": 50000,      -- Large cache: ~150 MB for 768-dim
  "mutationsBeforeRebuild": 50,      -- Frequent rebuilds for freshness
  "storeVectorsInGraph": true        -- Enable for large indexes to reduce cache pressure
};

CREATE INDEX ON Document (embedding) LSM_VECTOR METADATA {
  "dimensions": 768,
  "similarity": "COSINE",
  "maxConnections": 12,
  "beamWidth": 80,
  "locationCacheSize": 100000,
  "graphBuildCacheSize": 10000,
  "mutationsBeforeRebuild": 300,     -- Reduce rebuild frequency for write throughput
  "storeVectorsInGraph": false       -- Disable: frequent updates make graph storage costly
};

CREATE INDEX ON Document (embedding) LSM_VECTOR METADATA {
  "dimensions": 768,
  "similarity": "COSINE",
  "maxConnections": 24,
  "beamWidth": 150,
  "locationCacheSize": 200000,       -- Larger cache for hot vectors
  "graphBuildCacheSize": 10000,
  "mutationsBeforeRebuild": 50,      -- Frequent rebuilds for fresh results
  "storeVectorsInGraph": true        -- Enable: search-heavy benefits from co-location
};

-- Configure global defaults
ALTER DATABASE `arcadedb.vectorIndex.locationCacheSize` 100000;
ALTER DATABASE `arcadedb.vectorIndex.graphBuildCacheSize` 10000;
ALTER DATABASE `arcadedb.vectorIndex.mutationsBeforeRebuild` 100;
ALTER DATABASE `arcadedb.vectorIndex.inactivityRebuildTimeoutMs` 15000;
ALTER DATABASE `arcadedb.vectorIndex.storeVectorsInGraph` false;

-- View current settings
SELECT FROM information_schema.settings
WHERE name LIKE 'arcadedb.vectorIndex.%';

RAM (MB) = cacheSize × (dimensions × 4 + 64) / 1024 / 1024

CREATE INDEX ON Document (embedding) LSM_VECTOR METADATA {
  "dimensions": 384,
  "similarity": "COSINE",
  "quantization": "NONE"  -- Default, can be omitted
};

CREATE INDEX ON Document (embedding) LSM_VECTOR METADATA {
  "dimensions": 384,
  "similarity": "COSINE",
  "quantization": "INT8"  -- Recommended for production
};

CREATE INDEX ON Document (embedding) LSM_VECTOR METADATA {
  "dimensions": 384,
  "similarity": "COSINE",
  "quantization": "BINARY"  -- Use when memory is severely constrained
};

// Compare recall between NONE and INT8
LSMVectorIndex indexNone = createIndex(quantization=NONE);
LSMVectorIndex indexInt8 = createIndex(quantization=INT8);

double recall = measureRecall(indexNone, indexInt8, testQueries, k=10);
// Expect: recall > 0.95 for INT8

CREATE INDEX ON Document (embedding) LSM_VECTOR METADATA {
  "dimensions": 1536,
  "similarity": "COSINE",
  "quantization": "INT8",           -- 4x memory reduction
  "storeVectorsInGraph": true,      -- Fast co-located access
  "locationCacheSize": 200000,
  "mutationsBeforeRebuild": 1000
};

Map<String, Long> stats = index.getStats();
System.out.println("Fetched from quantized: " + stats.get("vectorFetchFromQuantized"));
System.out.println("Fetched from documents: " + stats.get("vectorFetchFromDocuments"));
// Should see: vectorFetchFromQuantized > 0, vectorFetchFromDocuments = 0

-- Same query works for all quantization types
SELECT expand(vectorNeighbors('Document[embedding]', $queryVector, 10))

// Stage 1: Fast binary search (top 100)
List<RID> candidates = binaryIndex.findNeighborsFromVector(query, 100);

// Stage 2: Rerank with full precision (top 10)
List<Result> finalResults = rerankWithFullPrecision(candidates, query, 10);

CREATE INDEX ON SmallDoc (embedding) LSM_VECTOR METADATA {
  "dimensions": 384,
  "similarity": "COSINE"
  -- quantization defaults to NONE (full precision)
};

CREATE INDEX ON MediumDoc (embedding) LSM_VECTOR METADATA {
  "dimensions": 768,
  "similarity": "COSINE",
  "quantization": "INT8"  -- 4x memory reduction, minimal accuracy loss
};

CREATE INDEX ON LargeDoc (embedding) LSM_VECTOR METADATA {
  "dimensions": 1536,
  "similarity": "COSINE",
  "quantization": "INT8",
  "storeVectorsInGraph": true,     -- Co-locate vectors with graph
  "locationCacheSize": 200000,     -- Cache hot vectors
  "mutationsBeforeRebuild": 1000   -- Reduce rebuild frequency
};

-- Create both indexes: BINARY for fast pre-filter, NONE for reranking
CREATE INDEX ON MassiveDoc (embeddingBinary) LSM_VECTOR METADATA {
  "dimensions": 1536,
  "similarity": "COSINE",
  "quantization": "BINARY",         -- 32x compression for fast search
  "storeVectorsInGraph": true
};

CREATE INDEX ON MassiveDoc (embedding) LSM_VECTOR METADATA {
  "dimensions": 1536,
  "similarity": "COSINE",
  "quantization": "NONE"            -- Full precision for reranking
};

CREATE INDEX ON EdgeDoc (embedding) LSM_VECTOR METADATA {
  "dimensions": 384,
  "similarity": "COSINE",
  "quantization": "BINARY",         -- Extreme compression (32x)
  "maxConnections": 8,              -- Reduce graph size
  "beamWidth": 100,
  "locationCacheSize": 50000,       -- Limit cache
  "graphBuildCacheSize": 5000,      -- Limit build cache
  "storeVectorsInGraph": true       -- Reduce I/O
};

// Create indexes with different quantization
LSMVectorIndex indexNone = createIndex("NONE", 1536);
LSMVectorIndex indexInt8 = createIndex("INT8", 1536);
LSMVectorIndex indexBinary = createIndex("BINARY", 1536);

// Generate test queries
List<float[]> testQueries = generateTestQueries(100);

// Measure recall against NONE baseline
for (float[] query : testQueries) {
  List<RID> groundTruth = indexNone.findNeighborsFromVector(query, 10);
  List<RID> int8Results = indexInt8.findNeighborsFromVector(query, 10);
  List<RID> binaryResults = indexBinary.findNeighborsFromVector(query, 10);

  double int8Recall = calculateRecall(int8Results, groundTruth);
  double binaryRecall = calculateRecall(binaryResults, groundTruth);

  System.out.printf("Query recall - INT8: %.2f%%, BINARY: %.2f%%\n",
                    int8Recall * 100, binaryRecall * 100);
}

// Measure memory usage
Map<String, Long> statsNone = indexNone.getStats();
Map<String, Long> statsInt8 = indexInt8.getStats();
Map<String, Long> statsBinary = indexBinary.getStats();

System.out.println("Memory estimates:");
System.out.println("  NONE:   " + statsNone.get("estimatedMemoryBytes") / 1024 / 1024 + " MB");
System.out.println("  INT8:   " + statsInt8.get("estimatedMemoryBytes") / 1024 / 1024 + " MB");
System.out.println("  BINARY: " + statsBinary.get("estimatedMemoryBytes") / 1024 / 1024 + " MB");

-- Returns rows with .record (full document) and .distance
SELECT expand(vectorNeighbors('Document[embedding]', $queryVector, 10))

-- Find the 10 nearest neighbors
SELECT expand(vectorNeighbors('Document[embedding]', $queryVector, 10))

-- Create multi-modal schema
CREATE VERTEX TYPE MultiModalRecord;
CREATE PROPERTY MultiModalRecord.imageEmbedding LIST OF FLOAT;
CREATE PROPERTY MultiModalRecord.textEmbedding LIST OF FLOAT;

-- Create separate indexes
CREATE INDEX ON MultiModalRecord (imageEmbedding) LSM_VECTOR METADATA {
  dimensions: 512,
  similarity: 'COSINE'
};

CREATE INDEX ON MultiModalRecord (textEmbedding) LSM_VECTOR METADATA {
  dimensions: 768,
  similarity: 'COSINE'
};

-- Query by image modality
SELECT expand(vectorNeighbors('MultiModalRecord[imageEmbedding]', [0.1, 0.2, ...], 5))

-- Query by text modality
SELECT expand(vectorNeighbors('MultiModalRecord[textEmbedding]', [0.2, 0.3, ...], 5))

-- Returns array of objects with distance and keys
SELECT vectorNeighbors('Document[embedding]', $queryVector, 10) AS neighbors;

-- Create parent type with vector index
CREATE VERTEX TYPE Embedding;
CREATE PROPERTY Embedding.vector ARRAY_OF_FLOATS;
CREATE INDEX ON Embedding (vector) LSM_VECTOR METADATA {
  dimensions: 768,
  similarity: 'COSINE'
};

-- Create specialized child types
CREATE VERTEX TYPE ImageEmbedding EXTENDS Embedding;
CREATE VERTEX TYPE DocumentEmbedding EXTENDS Embedding;
CREATE VERTEX TYPE AudioEmbedding EXTENDS Embedding;

SELECT expand(vectorNeighbors('ImageEmbedding[vector]', $queryVector, 50))

SELECT expand(vectorNeighbors('Embedding[vector]', $queryVector, 50))

import com.arcadedb.database.Database;
import com.arcadedb.schema.Schema;
import com.arcadedb.schema.Type;
import com.arcadedb.index.lsm.LSMVectorIndex;
import com.arcadedb.index.lsm.LSMVectorIndexBuilder;
import com.arcadedb.index.vector.VectorSimilarityFunction;

// Get or create schema
final Schema schema = database.getSchema();
if (!schema.existsType("Document")) {
  schema.createVertexType("Document");
}

// Create vector property
if (!schema.existsProperty("Document", "embedding")) {
  schema.createProperty("Document", "embedding", Type.ARRAY_OF_FLOATS);
}

// Create LSMVectorIndex with builder pattern
final LSMVectorIndexBuilder builder = new LSMVectorIndexBuilder(
    database,
    "Document",
    new String[]{"embedding"})
    .withDimensions(768)
    .withSimilarity(VectorSimilarityFunction.COSINE)
    .withMaxConnections(16)
    .withBeamWidth(100)
    .withIdProperty("id")
    .withAddHierarchy(false)          // Enable for large/complex datasets (100K+)
    .withStoreVectorsInGraph(false);  // Enable for large indexes (100K+)

final LSMVectorIndex index = builder.create();

// Get existing index
final LSMVectorIndex index = (LSMVectorIndex) database.getSchema()
    .getIndexByName("Document[embedding]");

// Add vectors with callback
index.addAll(embeddings, (vertex, item, total) -> {
  // Optional callback for handling vertex creation
  // Useful for cascading operations like creating relationships
  if (vertex != null) {
    // Create relationships, update metadata, etc.
    System.out.println("Indexed vertex " + vertex.getIdentity() +
                      " (" + item + "/" + total + ")");
  }
});

import com.arcadedb.utility.JSONObject;

final JSONObject metadata = new JSONObject()
    .put("dimensions", 768)
    .put("similarity", "COSINE")
    .put("maxConnections", 16)
    .put("beamWidth", 100)
    .put("idPropertyName", "id")
    .put("addHierarchy", false)         // Enable for large/complex datasets (100K+)
    .put("storeVectorsInGraph", false)  // Enable for large indexes (100K+)
    .put("locationCacheSize", 100000)   // Optional: limit cache size
    .put("graphBuildCacheSize", 10000)  // Optional: limit build cache
    .put("mutationsBeforeRebuild", 100); // Optional: rebuild frequency

final LSMVectorIndexBuilder builder = new LSMVectorIndexBuilder(
    database,
    "Document",
    new String[]{"embedding"})
    .fromMetadata(metadata);

final LSMVectorIndex index = builder.create();

import com.arcadedb.query.sql.executor.ResultSet;

// Define an example query vector. Replace with your actual vector.
float[] queryVector = new float[] {0.1f, 0.2f, 0.3f}; // Vector must match index dimensions

// Perform similarity search using SQL
final ResultSet resultSet = database.query("sql",
    "SELECT expand(vectorNeighbors('Document[embedding]', ?, 10))",
    queryVector);

while (resultSet.hasNext()) {
  final Result result = resultSet.next();
  // Each result has .record (the document) and .distance
  System.out.println("Similar document: " + result.toJSON());
}

database.transaction(() -> {
  // Create vertices with embeddings
  final MutableVertex vertex = database.newVertex("Document");
  vertex.set("content", "Sample text");

  // In a real application, this vector would be generated by an embedding model
  // and should match the dimensions of the index.
  float[] embeddingVector = new float[] {0.5f, 0.6f, 0.7f};
  vertex.set("embedding", embeddingVector);
  vertex.save();

  // Index is automatically updated on transaction commit
});

import com.arcadedb.index.vector.HnswVectorIndexRAM;
import com.arcadedb.index.vector.VectorSimilarityFunction;
import com.arcadedb.index.vector.Item;
import com.arcadedb.schema.Type;
import java.util.Collection;
import java.util.List;

// For bulk loading only - use LSMVectorIndex for production
String indexName = "Document[embedding]";
if (!database.getSchema().existsIndex(indexName)) {
  // Define parameters for the index
  int dimensions = 768;
  VectorSimilarityFunction distanceFunction = VectorSimilarityFunction.COSINE;
  int m = 16;
  int ef = 200;
  int efConstruction = 200;
  // Example embeddings. In a real scenario, this would be a large collection.
  Collection<Item<Object, float[]>> embeddings = List.of(
      new Item<>("doc1", new float[]{0.1f, 0.2f}),
      new Item<>("doc2", new float[]{0.3f, 0.4f})
  );

  // Step 1: Load into RAM-based index
  final HnswVectorIndexRAM<Object, float[], Item<Object, float[]>, Float> hnswIndex =
      HnswVectorIndexRAM.newBuilder(dimensions, distanceFunction, 100_000)
          .withM(m)
          .withEf(ef)
          .withEfConstruction(efConstruction)
          .build();

  hnswIndex.addAll(embeddings,
                   Runtime.getRuntime().availableProcessors(), null);

  // Step 2: Create persistent index
  hnswIndex.createPersistentIndex(database)
      .withVertexType("Document")
      .withEdgeType("Proximity")
      .withVectorProperty("embedding", Type.ARRAY_OF_FLOATS)
      .withIdProperty("id")
      .create();
}

-- Create index for document embeddings
CREATE VERTEX TYPE Document;
CREATE PROPERTY Document.content STRING;
CREATE PROPERTY Document.embedding LIST OF FLOAT;
CREATE INDEX ON Document (embedding) LSM_VECTOR METADATA {
  "dimensions": 384,
  "similarity": "COSINE"
};

-- Query for the 10 nearest documents
SELECT expand(vectorNeighbors('Document[embedding]', $queryEmbedding, 10))

-- Create index for image feature vectors
CREATE VERTEX TYPE Image;
CREATE PROPERTY Image.url STRING;
CREATE PROPERTY Image.features LIST OF FLOAT;
CREATE INDEX ON Image (features) LSM_VECTOR METADATA {
  dimensions: 512,
  similarity: 'COSINE'
};

-- Find the 5 most similar images
SELECT expand(vectorNeighbors('Image[features]', $imageFeatures, 5))

-- Create index for user/item embeddings
CREATE VERTEX TYPE Product;
CREATE PROPERTY Product.embedding LIST OF FLOAT;
CREATE INDEX ON Product (embedding) LSM_VECTOR METADATA {
  "dimensions": 128,
  "similarity": "COSINE"
};

-- Find the 20 most similar products for recommendations
SELECT expand(vectorNeighbors('Product[embedding]', $userPreferenceVector, 20))