Pinecone Integration with Graphbit¶
Overview¶
This guide explains how to connect Pinecone to Graphbit, enabling you to perform vector similarity search over high-dimensional embeddings generated by LLMs. With this integration, you can store, index, and search embeddings for tasks like semantic search, retrieval-augmented generation, and more.
Prerequisites¶
- Pinecone API Key: Obtain from Pinecone Console.
- OpenAI API Key: For LLM and embedding generation (or another supported embedding provider).
- Graphbit installed and configured (see installation guide).
- Python environment with
pinecone
,graphbit
, and optionallypython-dotenv
installed. - .env file in your project root with the following variables:
Step 1: Initialize Pinecone¶
Set up Pinecone and Graphbit, and ensure the index exists and is ready:
import os
import time
import uuid
from pinecone import Pinecone, ServerlessSpec
from dotenv import load_dotenv
load_dotenv()
INDEX_NAME = "graphbit-vector"
PINECONE_API_KEY = os.getenv("PINECONE_API_KEY")
pinecone_client = Pinecone(api_key=PINECONE_API_KEY)
index_list = pinecone_client.list_indexes()
pinecone_client.create_index(
name=INDEX_NAME,
vector_type="dense",
dimension=1536,
metric="cosine",
spec=ServerlessSpec(cloud="aws", region="us-east-1"),
)
index = pinecone_client.Index(INDEX_NAME)
Step 2: Generate and Upsert Embeddings¶
Use Graphbit's embedding client to generate embeddings and upsert them into Pinecone:
from graphbit import EmbeddingConfig, EmbeddingClient
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
EMBEDDING_MODEL = "text-embedding-3-small"
embedding_config = EmbeddingConfig.openai(model=EMBEDDING_MODEL, api_key=OPENAI_API_KEY)
embedding_client = EmbeddingClient(embedding_config)
texts = [
"GraphBit is a framework for LLM workflows and agent orchestration.",
"Pinecone enables vector search over high-dimensional embeddings.",
"OpenAI offers tools for LLMs and embeddings."
]
embeddings = embedding_client.embed_many(texts)
# Prepare vectors for upsert: (id, values, metadata)
vectors = [
(str(uuid.uuid4()), emb, {"text": txt})
for emb, txt in zip(embeddings, texts)
]
upsert_response = index.upsert(vectors=vectors)
print("Upsert response:", upsert_response)
Step 3: Perform Similarity Search¶
Embed your query and search for similar vectors in Pinecone:
query = "What is GraphBit?"
embed_query = embedding_client.embed(query)
print("Query embedding shape:", len(embed_query))
# Standard Pinecone query
resp = index.query(vector=embed_query, top_k=2, include_metadata=True)
print("Query response:", resp)
# Pretty-print results
for match in resp["matches"]:
print(f"Score: {match['score']:.4f}")
print(f"Text: {match['metadata']['text']}")
print("---")
Full Example¶
import os
import time
import uuid
from pinecone import Pinecone, ServerlessSpec
from graphbit import EmbeddingConfig, EmbeddingClient
INDEX_NAME = "graphbit-vector"
PINECONE_API_KEY = os.getenv("PINECONE_API_KEY")
pinecone_client = Pinecone(api_key=PINECONE_API_KEY)
index_name = INDEX_NAME
index_list = pinecone_client.list_indexes()
# Check if the index exists
if index_name not in [idx["name"] for idx in index_list]:
print(f"Index {index_name} does not exist. Creating it...")
pinecone_client.create_index(
name=index_name,
vector_type="dense",
dimension=1536,
metric="cosine",
spec=ServerlessSpec(cloud="aws", region="us-east-1"),
)
# Wait for index to be ready
while True:
status = pinecone_client.describe_index(index_name)
if status["status"]["ready"]:
break
print("Waiting for index to be ready...")
time.sleep(2)
index = pinecone_client.Index(index_name)
else:
index = pinecone_client.Index(index_name)
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
EMBEDDING_MODEL = "text-embedding-3-small"
embedding_config = EmbeddingConfig.openai(model=EMBEDDING_MODEL, api_key=OPENAI_API_KEY)
embedding_client = EmbeddingClient(embedding_config)
text = ["GraphBit is a framework for LLM workflows and agent orchestration.", "Pinecone enables vector search over high-dimensional embeddings.", "OpenAI offers tools for LLMs and embeddings."]
embeddings = embedding_client.embed_many(text)
vectors = [(str(uuid.uuid4()), emb, {"text": txt}) for emb, txt in zip(embeddings, text)]
upsert_response = index.upsert(vectors=vectors)
print("Upsert response:", upsert_response)
# Waiting until the vector is available in the index
NAMESPACE = "__default__"
attempt = 0
target_id = vectors[0][0]
while True:
response = index.fetch(ids=[target_id], namespace=NAMESPACE)
if response.vectors and target_id in response.vectors:
print(f"Confirmed upsert: vector {target_id} is available.")
break
print(f"Waiting for upsert completion... attempt {attempt}")
attempt += 1
time.sleep(2)
query = "What is GraphBit?"
embed_query = embedding_client.embed(query)
print("Query embedding shape:", len(embed_query))
results = index.query(vector=embed_query, top_k=2, include_metadata=True)
print(results)
# Pretty-print results
for match in results["matches"]:
print(f"Score: {match['score']:.4f}")
print(f"Text: {match['metadata']['text']}")
print("---")
This integration enables you to leverage Graphbit's embedding capabilities with Pinecone's vector database for scalable, production-grade semantic search and retrieval workflows.