Integrating OpenAI's Retrieval-Augmented Generation in NET Applications

dotnet add package OpenAI
dotnet add package PdfPig
dotnet add package Microsoft.Extensions.AI
dotnet add package Microsoft.Extensions.AI.OpenAI

using PdfPig;

// Load PDF
using var pdfDocument = PdfDocument.Open("path/to/your/document.pdf");

// Extract text
var text = string.Join("\n", pdfDocument.GetPages().Select(p => p.GetText()));

// Split text into chunks (e.g., paragraphs)
var chunks = text.Split(new[] { "\n\n" }, StringSplitOptions.RemoveEmptyEntries);

using OpenAI;

// Initialize OpenAI client with API key
var openAiClient = new OpenAIClient(Environment.GetEnvironmentVariable("OPENAI_API_KEY"));

// Generate embeddings for each chunk
foreach (var chunk in chunks)
{
    var embedding = await openAiClient.Embeddings.GenerateAsync(chunk, "text-embedding-ada-002");
    // Store the embedding in a local database or in-memory structure
}

using System.Collections.Generic;

// In-memory database example
var embeddingDatabase = new Dictionary();

// Store embeddings
foreach (var chunk in chunks)
{
    var embedding = await openAiClient.Embeddings.GenerateAsync(chunk, "text-embedding-ada-002");
    embeddingDatabase.Add(chunk, embedding.Vector.ToArray());
}

using Microsoft.Extensions.AI;

// Initialize chat client
var chatClient = new OpenAIClient(Environment.GetEnvironmentVariable("OPENAI_API_KEY")).AsChatClient();

// Chat loop
while (true)
{
    Console.Write("Enter your question: ");
    var query = Console.ReadLine();

    if (query == "exit")
        break;

    // Generate query embedding
    var queryEmbedding = await openAiClient.Embeddings.GenerateAsync(query, "text-embedding-ada-002");

    // Perform vector similarity search
    var similarities = embeddingDatabase.Select(kvp => (Key: kvp.Key, Similarity: CosineSimilarity(queryEmbedding.Vector.ToArray(), kvp.Value)));

    // Get top matches
    var topMatches = similarities.OrderByDescending(s => s.Similarity).Take(3);

    // Display matches
    foreach (var match in topMatches)
    {
        Console.WriteLine($"Match: {match.Key}, Similarity: {match.Similarity:F2}");
    }
}

// Cosine similarity function
float CosineSimilarity(float[] vec1, float[] vec2)
{
    var dotProduct = vec1.Zip(vec2, (a, b) => a * b).Sum();
    var magnitude1 = MathF.Sqrt(vec1.Sum(x => x * x));
    var magnitude2 = MathF.Sqrt(vec2.Sum(x => x * x));
    return dotProduct / (magnitude1 * magnitude2);
}