Indexer - Milvus v2 (Recommended)

go get github.com/cloudwego/eino-ext/components/indexer/milvus2

package main

import (
        "context"
        "log"
        "os"

        "github.com/cloudwego/eino-ext/components/embedding/ark"
        "github.com/cloudwego/eino/schema"
        "github.com/milvus-io/milvus/client/v2/milvusclient"

        milvus2 "github.com/cloudwego/eino-ext/components/indexer/milvus2"
)

func main() {
        // Get environment variables
        addr := os.Getenv("MILVUS_ADDR")
        username := os.Getenv("MILVUS_USERNAME")
        password := os.Getenv("MILVUS_PASSWORD")
        arkApiKey := os.Getenv("ARK_API_KEY")
        arkModel := os.Getenv("ARK_MODEL")

        ctx := context.Background()

        // Create embedding model
        emb, err := ark.NewEmbedder(ctx, &ark.EmbeddingConfig{
                APIKey: arkApiKey,
                Model:  arkModel,
        })
        if err != nil {
                log.Fatalf("Failed to create embedding: %v", err)
                return
        }

        // Create indexer
        indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{
                ClientConfig: &milvusclient.ClientConfig{
                        Address:  addr,
                        Username: username,
                        Password: password,
                },
                Collection:   "my_collection",

                Vector: &milvus2.VectorConfig{
                        Dimension:  1024, // Match embedding model dimension
                        MetricType: milvus2.COSINE,
                        IndexBuilder: milvus2.NewHNSWIndexBuilder().WithM(16).WithEfConstruction(200),
                },
                Embedding:    emb,
        })
        if err != nil {
                log.Fatalf("Failed to create indexer: %v", err)
                return
        }
        log.Printf("Indexer created successfully")

        // Store documents
        docs := []*schema.Document{
                {
                        ID:      "doc1",
                        Content: "Milvus is an open-source vector database",
                        MetaData: map[string]any{
                                "category": "database",
                                "year":     2021,
                        },
                },
                {
                        ID:      "doc2",
                        Content: "EINO is a framework for building AI applications",
                },
        }
        ids, err := indexer.Store(ctx, docs)
        if err != nil {
                log.Fatalf("Failed to store: %v", err)
                return
        }
        log.Printf("Store success, ids: %v", ids)
}

indexBuilder := milvus2.NewHNSWIndexBuilder().
        WithM(16).              // Maximum connections per node (4-64)
        WithEfConstruction(200) // Search width during index construction (8-512)

indexBuilder := milvus2.NewIVFFlatIndexBuilder().
        WithNList(256) // Number of cluster units (1-65536)

indexBuilder := milvus2.NewIVFPQIndexBuilder().
        WithNList(256). // Number of cluster units
        WithM(16).      // Number of sub-quantizers
        WithNBits(8)    // Bits per sub-quantizer (1-16)

indexBuilder := milvus2.NewSCANNIndexBuilder().
        WithNList(256).           // Number of cluster units
        WithRawDataEnabled(true)  // Enable raw data for reranking

indexBuilder := milvus2.NewDiskANNIndexBuilder() // Disk-based, no additional parameters

indexBuilder := milvus2.NewSparseInvertedIndexBuilder().
        WithDropRatioBuild(0.2) // Ratio of small values to ignore during build (0.0-1.0)

indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{
    // ... basic configuration ...
    Collection:        "hybrid_collection",
    
    Sparse: &milvus2.SparseVectorConfig{
        VectorField: "sparse_vector",
        MetricType:  milvus2.BM25, 
        // Method defaults to SparseMethodAuto when using BM25
    },
    
    // Analyzer configuration for BM25
    FieldParams: map[string]map[string]string{
        "content": {
            "enable_analyzer": "true",
            "analyzer_params": `{"type": "standard"}`, // Use {"type": "chinese"} for Chinese
        },
    },
})

indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{
    Collection: "sparse_collection",
    
    Sparse: &milvus2.SparseVectorConfig{
        VectorField: "sparse_vector",
        MetricType:  milvus2.IP,
        Method:      milvus2.SparseMethodPrecomputed,
    },
})

// Store documents with sparse vectors
doc := &schema.Document{ID: "1", Content: "..."}
doc.WithSparseVector(map[int]float64{
    1024: 0.5,
    2048: 0.3,
})
indexer.Store(ctx, []*schema.Document{doc})

// Create indexer without embedding
indexer, err := milvus2.NewIndexer(ctx, &milvus2.IndexerConfig{
    ClientConfig: &milvusclient.ClientConfig{
        Address: "localhost:19530",
    },
    Collection:   "my_collection",
    Vector: &milvus2.VectorConfig{
        Dimension:  128,
        MetricType: milvus2.L2,
    },
    // Embedding: nil, // Leave empty
})

// Store documents with precomputed vectors
docs := []*schema.Document{
    {
        ID:      "doc1",
        Content: "Document with existing vector",
    },
}

// Attach dense vector to document
// Vector dimension must match collection dimension
vector := []float64{0.1, 0.2, ...} 
docs[0].WithDenseVector(vector)

// Attach sparse vector (optional, if Sparse is configured)
// Sparse vector is a mapping of index -> weight
sparseVector := map[int]float64{
    10: 0.5,
    25: 0.8,
}
docs[0].WithSparseVector(sparseVector)

ids, err := indexer.Store(ctx, docs)