Demo source code

The core of demo.cs, the C# source of the multi-file document Q&A demo shipped with the toolkit, arranged by stage. Windows Forms plumbing lives in a separate demo_ui.cs and is omitted here; the complete project is included in the evaluation download. The tuning values shown (retrieval percentage, enrichment budget, coverage share, confidence threshold) are examples and should be adjusted for your documents and token budget.

The demo is provided as sample code for building your own product. See the license agreement in the Documentation tab regarding distribution of the demo itself.

Setup: fields, per-file state and model initialization

The class-level state of the demo: collection totals, tuning percentages, the per-file ClsFiles record that holds each document's chunks, metadata and Ven object, and the initialization of the embedding and chat clients.

using LangChain.Databases;      // comes from LangChain.Databases.InMemory
using LangChain.Databases.Sqlite;
using LangChain.DocumentLoaders; 
using LangChain.Extensions; // Required for the unified VectorStore type
using LangChain.Providers;  // For OpenAI provider specific types
using LangChain.Providers.OpenAI;
using Ollama;
using OpenAI.Chat;   // official OpenAI SDK to bypass LangChain when sending the chunks to LLM
using OpenAI.VectorStores;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.Diagnostics.Eventing.Reader;
using System.Diagnostics.SymbolStore;
using System.Drawing.Drawing2D;
using System.IO;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Security.Policy;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Windows.Forms;
using tryAGI.OpenAI;
using static System.Windows.Forms.VisualStyles.VisualStyleElement.ProgressBar;

using SubSystems.RagDocumentToolkit.Dan;
using SubSystems.RagDocumentToolkit.Ven;


namespace Demo
{
    public partial class Demo : Form
    {
string msg = "";
IReadOnlyCollection<Document>? AllDocs = null;   // Document object containing all documents - used for
                                                 // retrieval across all documents in the collection
string TokenMessage = "";            // number of tokens used message
int TotalTokenCount = 0;             // token count for all files in the collection 
int TotalChunkCount = 0;             // chunk count for all files in the collection 
int TokenThreshold = 20000;          // when total token count is below this threshold, we send all file chunks to LLM for each user query.
                                     // Above this threshold, we apply user-query-dependent retrieval and Vector Enrichment to
                                     // determine only the relevant chunks to send to LLM
int retrievalPct = 15;               // retrieval search limit as the percentage of total chunks in the file collection 
int enrichmentPct = 10;              // enrichment budget as the percentage of total tokens in the file collection
int globalSpreadPct = 25;            // for broad (global) query, percent of retrieval chunks to distribute to each file in file-size proportion
int filterTokenPct = 10;             // Filter routing token allowance
int filterTokenCount = 0;            // filter token count

int TotalRetrievalTokens = 0;        // number of tokens in the retrieved set for the query

string IndexDoc = "";
int TotalFilterTokens = 0;           // number of tokens in the combined phrase document
bool AnswerSummaryQueryUsingPhraseDoc = true;  // true: use the phrase doc to answer a collection-wide summary query
bool GlobalQueryExecutedUsingPhraseDoc = false;  // true if the global query is executed using the phrase document

string newUserQuestion = "";         // current user question

bool globalQuery = false;              // true if the current query is of a broad nature


string fullContent;
LangChain.Providers.Message userMessage;

// LangChain parameters for strategy evaluation and search
OpenAiProvider provider;

OpenAI.Chat.ChatClient? fullChat = null;
OpenAI.Chat.ChatClient? miniChat = null;


private List<LangChain.Providers.Message> chatHistory = new List<LangChain.Providers.Message>();
private List<LangChain.Providers.Message> filterChatHistory = new List<LangChain.Providers.Message>();

string htmlResponse = "", PrevHtmlResponse = "";
//private SqLiteVectorDatabase vectorDatabase;
private IVectorCollection? vectorCollection = null;   // collection containing all documents

private OpenAiEmbeddingModel embeddingsProvider;
private const string EmbeddingModelName = "text-embedding-3-small"; // if we switch to another AI, say Claude, both the 
                                                                    // EmbeddingModelName and the Dimensions (below)
                                                                    // need to change
int Dimensions = 1536;    // dimension of each LLM vector point.  1536 = recommended for ChatGPT
String CollectionName = "Dan_Docs_" + EmbeddingModelName.Replace("-", "_");  // in case of a persisted vector (This demo 
                                                                             // uses in-memory vectors), adding the embedding model to the
                                                                             // collection name will ensure that any old persisted
                                                                             // vector is not used for AI query

private OpenAiChatSettings Settings;
private int ChunkCount = 0;



private bool UserQueryCheckedForRevisions = false;   // true=checked the user query for Revisions
private bool QueryIsAboutRevisions = false;          // true=the query is about revisions (redlined text)
private bool UserQueryCheckedForComments = false;    // true=checked the user query for Commented text
private bool QueryIsAboutComments = false;           // true=the query is about commented text

private bool AiWorking = false;
private bool AiInitialized = false;

internal class ClsFiles
{
    internal string FilePath = "";        // file path of this document
    internal string DocId = "";           // unique document id of this document
    internal string DocTitle = "";        // document title

    internal Ven? ven = null;             // Instance of Vector Enrichment Library for this document
    internal int ChunkCount = 0;          // size of the chunk/MetaRec arrays
    internal string[]? chunk = null;      // chunk array
    internal Dictionary<string, object>[]? MetaRec = null;
    internal string KeyPhrases;           // key phrases for the document

    internal SortedSet<int> RetrieveSeq;  // retrieved sequences
    internal SortedSet<int> EnrichedSeq;  // enriched sequences
    internal int TokenBudget;             // Enrichment token budget for this document

    internal IReadOnlyCollection<Document>? docs = null; // chunk/meta-rec collection for this document - used for retrieval at the document level
    internal SqLiteVectorDatabase? vectorDatabase = null;
    internal IVectorCollection? vectorCollection = null;  // per-file vector collection

    internal int FirstIdx = 0;            // index of the first chunk/MetaRec record position within the combined document.
                                          // It would be the sum of ChunkCounts of the preceding files.

    internal bool HasTables = false;      // true=this file includes tables
    internal bool HasRevisions = false;   // true=this file includes revised text
    internal bool HasComments = false;    // true=this file includes comments
    internal int TokenCount = 0;          // token count for this file

    internal int RetrievalLimit = 0;      // number of records to retrieve for this document
    internal int ErichmentBudget = 0;     // Enrichment token count budget for this document

    internal int RetrievalTokenCount;     // The number of tokens in the retrieved set
    internal int EnrichmentTokenCount;    // The number of tokens added for enrichment

    internal int quota;                   // this file's quota for vector search pass #2

    internal bool InFilter;               // true if filtering is enabled and this file is included after filtering for a user query
    internal string PhraseDoc = "";       // phrase document - key values/phrases from the meta information about this file, in suitable chunks to send to AI
    internal int FilterTokens = 0;        // number of tokens for the PhraseDoc
}

List<ClsFiles> files = new List<ClsFiles>();   // current file collection

Dictionary<string, ClsFiles>? filesDict = null;

internal class ClsFilterResult
{
    internal bool FilterApplied = false;        // the filter query set the InFilter flag for those documents found relevant after the filter query
    internal List<LangChain.Providers.Message>? payload = null;   // payload already created and ready to be sent to AI
}

public Demo()
{
    InitializeComponent();
}

private async void Demo_Load(object sender, EventArgs e)
{
    await InitUi(sender, e);

    if (Dan.DasHasSubSystemsOpenAIEvalKeyExpired())     // Sub Systems' OpenAI key expired for your eval, please provide your own key
    {
        LblStatus.Text = "Sub Systems' OpenAI key expired for your eval.  Please enter your OpenAI key to proceed:";
        FldKey.Visible = true;
        FldAcceptKey.Visible = true;    
        FldSend.Enabled = false;
        FldAddDoc.Enabled = false;
        FldRestore.Enabled = false;
        return;
    }

    await InitAi();
}

private async Task InitAi()
{
    string fullModel = "gpt-5.6-terra";   // model for main chat
    string miniModel = "gpt-5.6-luna";    // model for index lookup or classification queries

    if (Dan.DasHasSubSystemsOpenAIEvalKeyExpired())  // you are using your own key because Sub Systems' OpenAI key has expired for your eval
    { 
        string YourOpenAiKey = FldKey.Text.Trim();
        provider = new OpenAiProvider(YourOpenAiKey);

        fullChat = new OpenAI.Chat.ChatClient(fullModel, YourOpenAiKey);
        miniChat = new OpenAI.Chat.ChatClient(miniModel, YourOpenAiKey);
    }
    else {   // Apply Sub Systems' OpenAI key, which is valid during your eval period
        Dan.DasSetSubSystemsOpenAIEvalKey(out provider, 
              out fullChat, out miniChat, fullModel, miniModel);
    }

    // Initialize the embedding model (used to mathematically index the document text)
    // Instantiate OpenAiEmbeddingModel using the provider and the target model
    embeddingsProvider = new OpenAiEmbeddingModel(provider, EmbeddingModelName);

    Settings = new OpenAiChatSettings
    {
        // OpenAI renamed 'MaxTokens' to 'MaxCompletionTokens' recently
        MaxCompletionTokens = 1000,    // don't let AI answers get too long.  Also works as a circuit breaker if AI gets caught in an internal loop interpreting our data
        Temperature = 1   // don't let AI get too general, stick to my document
    };

    // check if the OpenAI key is valid
    try
    {
        // Make a tiny, minimal request to verify the key
        await miniChat.CompleteChatAsync("ping");
        // If it reaches here without throwing a 401 exception, the key is correct!
    }
    catch (System.ClientModel.ClientResultException ex) when (ex.Status == 401)
    {
        // Key is explicitly bad or unauthorized
        LblStatus.Text = "Invalid OpenAI API Key provided, Please enter your correct key";
        return;
    }
    catch (Exception ex)
    {
        // Handle other errors (like network/timeout issues)
        LblStatus.Text = $"Connection error: {ex.Message}";
        return;
    }

    FldSend.Enabled = true;
    AiInitialized = true;
    FldKey.Visible = false;
    FldAcceptKey.Visible = false;
    LblStatus.Text = "";
    FldAddDoc.Enabled = true;
    FldRestore.Enabled = true;
}

Adding documents with Dan and building the vector collections

Importing each file, calling DasGetMarkdown, creating the per-document Ven object, and loading chunks with their metadata into per-file and collection-wide vector stores.

/**************************************************************
 *  AddDocument:
 *  Add the given file to the document collection
 *  ***********************************************************/
async Task<bool> AddDocument(string NewFile)
{
    msg = "";  // reset any error message

    ClsFiles file = new ClsFiles();
    file.FilePath = NewFile;

    // Create markdown object
    Dan dan = new Dan();
    dan.LogMsg += LogDanMsg;
    dan.MdoNames += MdoNames;  // The library sends plain text for each chunk,
                               // so if your application needs to, it can detect the names used in the chunk text and
                               // return those names.  This can significantly increase the document index performance.

    SetStatusText($"Reading file: {NewFile}");

    // import the file into the Dan object
    if (!dan.DasImportFile(NewFile, GetDocType(NewFile)))
    {
        if (msg == "") LblStatus.Text = "Error getting markdown for: " + NewFile;
        else LblStatus.Text = msg;
        return false;
    }

    SetStatusText("Creating markdown...");


    // Get the markdown for the imported file
    Dan.DanResult mdo = dan.DasGetMarkdown(-1, -1); // -1 = get markdown for the entire document
    if (mdo == null)
    {
        if (msg == "") LblStatus.Text = "Error getting markdown for: " + NewFile;
        else LblStatus.Text = msg;
        return false;
    }

    SetStatusText("");

    file.DocId = mdo.DocId;    // Unique id for the document
    file.DocTitle = mdo.DocTitle;  // document title retrieved from the document. If the document title is not found, this field contains the document file path
    file.chunk = mdo.chunks;
    file.MetaRec = mdo.MetaRecs;
    file.ChunkCount = file.chunk.Length;

    // create the Vector Enrichment object for this markdown
    file.ven = null;
    try
    {
        file.ven = new Ven(file.MetaRec);
    }
    catch (Exception ex)
    {
        LblStatus.Text = "Error creating Vector Enrichment object: " + ex.ToString();
        return false;
    }
    file.HasTables = file.ven.VesHasTables();
    file.HasRevisions = file.ven.VesHasRevisions();
    file.HasComments = file.ven.VesHasComments();
    file.TokenCount = file.ven.VesGetTokenCount();

    file.PhraseDoc = "";   // the phrase document contains a summary of the meta records in a format suitable for AI query
    file.FilterTokens = 0;

    // build List<Document>() for this document for retrieval 
    // specific to this document
    List<Document> docs = new List<Document>();
    int ChunkCount = file.ChunkCount;
    for (int i = 0; i < ChunkCount; i++)
    {
        docs.Add(new Document(
            content: file.chunk[i],
            metadata: file.MetaRec[i]
        ));
    }

    file.docs = docs;   // assign to read-only collection

    file.vectorDatabase = new SqLiteVectorDatabase(dataSource: ":memory:");
    file.vectorCollection = await file.vectorDatabase.GetOrCreateCollectionAsync(
        collectionName: CollectionName + files.Count,  // added count to make a unique name, and differentiate from the global CollectionName
        dimensions: Dimensions     // 1536-dimensional vectors, this is a standard chosen by OpenAI; needs to change for other providers
    );

    await file.vectorCollection.AddDocumentsAsync(embeddingsProvider, docs);


    files.Add(file);

    ResetChat();   // reset chat related variables

    FldSave.Enabled = true;

    return true;
}

/**************************************************************
 * RecreateVectorStore:
 * Recreate the combined document object
 * ************************************************************/
async Task RecreateVectorStore()
{
    if (!AiInitialized) await InitAi();

    if (files.Count == 0)   // for an empty file collection
    {
        AllDocs = null;
        TotalTokenCount = 0;
        TotalChunkCount = 0;
        LblTokenCount.Text = "";
        return;
    }

    vectorCollection = await GetVectorCollection(true);   // collection containing all documents in the filter

    //clear chat history
    chatHistory = new List<LangChain.Providers.Message>();
    FldHtml.NavigateToString("");

    LblTokenCount.Text = $"Total Tokens: {(TotalTokenCount / 1000):N0}K";
    LblChunkCount.Text = $"Total Chunks: {TotalChunkCount}";
}

/*****************************************************************************
 * GetVectorCollection:
 * Get the vector collection for all files, or
 * for only the files in the preceding filter set.
 * When collecting all files, the method also updates TotalChunkCount, TotalTokenCount
 * and each file's beginning chunk index
 * **************************************************************************/
private async Task<IVectorCollection> GetVectorCollection(bool all)
{

    var docList = new List<Document>();
    int FirstIdx = 0;  // index of a document's first chunk/MetaRec in the combined collection

    // check if all files are selected
    if (!all)
    {
        ClsFiles file = files.FirstOrDefault(x => (!x.InFilter));   // find one file not in the filter 
        if (file == null) all = true;
    }

    if (all && vectorCollection != null && filesDict != null) return vectorCollection; // we already have it

    if (all)
    {
        TotalTokenCount = 0;
        TotalChunkCount = 0;

        filesDict = files.ToDictionary(f => f.DocId);  // look up file by DocId
    }

    foreach (ClsFiles file in files)
    {
        if (file.chunk == null || file.MetaRec == null || file.docs == null) continue;
        if (!all && !file.InFilter) continue;   // only include files in the filter 

        List<Document> FileDoc = file.docs as List<Document>;
        if (all || file.InFilter) docList.AddRange(FileDoc);

        if (all)  // save to class variables
        {
            file.FirstIdx = FirstIdx;
            FirstIdx += file.ChunkCount;  // FirstIdx of the next file

            TotalTokenCount += file.TokenCount;
            TotalChunkCount += file.ChunkCount;
        }
    }

    IReadOnlyCollection<Document> docs = docList;   // Document object containing all/filtered documents


    // Load the document markdown chunks directly into in-memory vector storage.
    // This takes our documents and builds a searchable index out of them.
    // Create the database in-memory (using the ":memory:" data source)
    SqLiteVectorDatabase vectorDatabase = new SqLiteVectorDatabase(dataSource: ":memory:");

    // Add the pre-split documents to the collection using the embedding provider
    // Note: OpenAI embeddings standard width is 1536 dimensions
    IVectorCollection vCollection = await vectorDatabase.GetOrCreateCollectionAsync(
        collectionName: CollectionName,
        dimensions: Dimensions     // 1536-dimensional vectors, this is a standard chosen by OpenAI; needs to change for other providers
    );

    await vCollection.AddDocumentsAsync(embeddingsProvider, docs);

    if (all)
    {
        AllDocs = docs;   // save to class variables
        vectorCollection = vCollection;   // If this included all files, save the collection for future queries
    }

    return vCollection;
}

/*******************************************************************
 * ResetChat:
 * Reset the chat related variables
 * *****************************************************************/
void ResetChat()
{
    AllDocs = null;    // nullify because it needs to be recreated using the updated set of files
    vectorCollection = null;
    chatHistory = new List<LangChain.Providers.Message>();
    filesDict = null;

    IndexDoc = "";  // combined phrase doc - needs to be recalculated
}
/********************************************************************

Name extraction hook

The MdoNames handler Dan calls with each chunk's plain text. This concept version uses regular expressions and word lists; a production application would use an NER model or its own domain vocabulary.

/********************************************************************
 * Here is a quick concept of extracting names of persons, places, and 
 * organizations.
 * In this concept check, we detect names and return them.
 * In your application you would use Microsoft.ML.OnnxRuntime, or another
 * such library, to detect names and return them. 
 * The 'ImportantTerms' can include checks for all important terms
 * for your organization or industry.
 * This step is not mandatory, but recommended.
 * *******************************************************************/
// Persons: honorific followed by 1-3 capitalized words ("Dr. Anita Patel")
static readonly Regex PersonRx = new Regex(
    @"\b(?:Dr|Mr|Mrs|Ms|Prof)\.?\s+((?:[A-Z][a-z]+\s?){1,3})",
    RegexOptions.Compiled);

// Orgs: 1-4 capitalized words ending in a corporate suffix ("CranePoint Manufacturing Co.")
static readonly Regex OrgRx = new Regex(
    @"\b((?:[A-Z][A-Za-z&]+\s+){1,4}(?:Inc\.?|LLC|Ltd\.?|Corp\.?|Co\.?|Group|Medical Center|Manufacturing))(?=[\s,.;:]|$)",
    RegexOptions.Compiled);

static readonly string[] Places = { "Austin", "St. Petersburg", "Port Haven", "Rm 118" };

static readonly HashSet<string> Terms = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
    { "Lisinopril", "Metformin", "Amlodipine", "Net-45", "Spindle bearing", "President" };

static bool ContainsWord(string text, string term) =>
    Regex.IsMatch(text, $@"\b{Regex.Escape(term)}\b", RegexOptions.IgnoreCase);

Dan.ClsMdoNames MdoNames(object sender, string DocTitle, int ChunkIndex, String text)
{
    var names = new Dan.ClsMdoNames();
    var persons = new HashSet<string>();
    var orgs = new HashSet<string>();
    var places = new List<string>();
    var terms = new List<string>();

    // Persons — capture group 1 is the name without the honorific
    foreach (Match m in PersonRx.Matches(text))
        persons.Add(m.Groups[1].Value.Trim());
    persons.Add("Jonathan");  // we add this just for our testing

    // Organizations — capture group 1 is the full org name
    foreach (Match m in OrgRx.Matches(text))
        orgs.Add(m.Groups[1].Value.Trim());

    // Places and domain terms — simple whole-word lookup against your lists
    foreach (var p in Places)
        if (ContainsWord(text, p)) places.Add(p);

    foreach (var t in Terms)
        if (ContainsWord(text, t)) terms.Add(t);

    names.PersonNames = persons.ToArray();
    names.OrgNames = orgs.ToArray();
    names.PlaceNames = places.ToArray();
    names.ImportantTerms = terms.ToArray();

    return names;
}

Answering a question: threshold check and routing

The send handler chooses between sending every chunk (small collections), the AI document filter, and retrieval with enrichment. IsGlobalQuery asks a small model whether the question is broad or pointed.

private async void FldSend_Click(object sender, EventArgs e)
{
    if (!AiInitialized) await InitAi();
    if (AiWorking) return;  // not done with the previous question

    AiWorking = true;   // don't take the next request until we are done with this one

    await FldHtml.EnsureCoreWebView2Async();

    // add the new question to chatHistory
    newUserQuestion = FldQuery.Text;
    chatHistory.Add(new LangChain.Providers.Message(newUserQuestion, LangChain.Providers.MessageRole.Human));

    PrevHtmlResponse = htmlResponse; // save the previous response
    UpdateChatBox(newUserQuestion, "<span id=\"progress\"></span>");
    LblStatus.Text = "";

    // get the payload to send to AI
    List<LangChain.Providers.Message>? payload = null;
    if (TotalTokenCount < TokenThreshold) payload = GetAllChunksContext();
    else
    {
        GlobalQueryExecutedUsingPhraseDoc = false;  // will be true if the user query is answered from the filter phrase document instead of the file collection.
                                                    // This could happen if filtering is enabled and AI determines that the user query is of a broad summary nature.
        payload = await GetContextForAboveThresholdQuery();
        if (GlobalQueryExecutedUsingPhraseDoc)
        {
            AiWorking = false;
            return;   // user query already executed using the phrase document as the source
        }
    }

    if (payload == null)
    {
        LblStatus.Text = "Error generating full AI payload.";
        AiWorking = false;
        return;
    }

    // send the payload to AI
    await SendToAI(fullChat, payload, chatHistory, true);
    AiWorking = false;
}

/************************************************************************
 * GetAllChunksContext:
 * Return the content of the chunks of the whole collection, plus the system prompt
 * and chatHistory
 * ********************************************************************/
private List<LangChain.Providers.Message> GetAllChunksContext()
{
    string chunks = "";

    foreach (ClsFiles file in files)
    {
        if (chunks.Length > 0) chunks += "\n\n";
        chunks += String.Join("\n\n", file.chunk);
    }

    // to take full advantage of caching, we place the static content before the chat history
    string dynamicSystemPrompt = Dan.DasGetSystemMessage() +
                 "\n\nUSE THIS EXTRACTED DOCUMENT CONTEXT TO ANSWER THE USER'S QUESTION:\n" +
                 chunks;

    // create a combined list: system message followed by the chat history
    LangChain.Providers.Message dynamicMessage = new LangChain.Providers.Message(dynamicSystemPrompt,
                                                                         LangChain.Providers.MessageRole.System);
    List<LangChain.Providers.Message> payload = [dynamicMessage, .. chatHistory];

    TokenMessage = $"Number of document tokens sent: {TotalTokenCount}";

    return payload;
}

/************************************************************************
 * GetContextForAboveThresholdQuery:
 * Get the content when the file collection has a token count above the threshold
 * below which we send all contents.
 * For this method we let LLM decide if the query is of a broad nature or if
 * it is a pointed query. We use this for further routing.
 * ********************************************************************/
private async Task<List<LangChain.Providers.Message>>? GetContextForAboveThresholdQuery()
{
    //var sw=StartStopwatch();

    // the following flags are set to true after evaluating the new question for revisions and comments
    UserQueryCheckedForRevisions = false;
    UserQueryCheckedForComments = false;

    GlobalQueryExecutedUsingPhraseDoc = false;

    // check if we need to do filtration routing 
    bool filterApplied = false;
    if (FldUseFilter.Checked)
    {
        ClsFilterResult result = await ApplyFilter();

        if (result.payload != null) return result.payload;

        if (result.FilterApplied)
        {  //  
            filterApplied = true;
            globalQuery = true;   // now treat this as a broad query over a narrow subset of files after filtration
        }

    }

    if (!filterApplied)
    {
        foreach (ClsFiles file in files) file.InFilter = true;  // assume all files in filter
        globalQuery = await IsGlobalQuery(newUserQuestion);     // Let AI determine the nature of the user query
    }
    //LogTime(sw, "after global query");

    return await GetRetrievalEnrichmentContext();
}

/******************************************************************************
    IsGlobalQuery:
    Determine the scope of the user question.
    This could be an expensive call.  We recorded it at 1600 ms
*******************************************************************************/
private async Task<bool> IsGlobalQuery(string userQuestion)
{
    try
    {
        // A quick, low-cost system prompt to categorize the question intent
        string routingPrompt = @"
       Analyze the user's question about a document. Categorize it into one of two strategies:
       - 'GLOBAL': The question requires aggregation, math, summaries across the whole file, or looking at data over many different pages (e.g., 'What is the average revenue?', 'Summarize the whole profile', 'List all names mentioned').
       - 'LOCAL': The question looks for a specific rule, fact, step, or localized piece of text (e.g., 'How do I modify the budget?', 'What is the subject's birthdate?').
       When in doubt, categorize as 'GLOBAL'.

       Respond with ONLY the word 'GLOBAL' or 'LOCAL'.";

        var activeChat = miniChat;
        if (FldUseFilter.Checked) activeChat = fullChat;   // when using the filter, leave miniChat just for filtering so filtering can execute with discounted cache pricing

        ChatCompletion decision = await activeChat.CompleteChatAsync(
            new List<ChatMessage> {
               new SystemChatMessage(routingPrompt),
               new UserChatMessage(userQuestion)
            },
            new ChatCompletionOptions { MaxOutputTokenCount = 50 });
        string answer = decision.Content[0].Text.Trim().ToUpperInvariant();
        return answer.StartsWith("GLOBAL");
    }
    catch (Exception)
    {
        LblStatus.Text = "Exception in IsGlobalQuery";
        return true;  // in the unlikely case of a crash, use the global strategy
    }
}

AI document filter: building the index and routing on it

Creating the collection index with VesCreateDocumentIndex, sending it with the question to the routing model using VesGetFilterSystemPrompt, and applying the confidence threshold to decide between whole-document, filtered and global modes.

/****************************************************************************
 * ApplyFilter:
 * When the filter is enabled, we use the combined phrase document to let AI
 * determine the subset of files that are needed to satisfy the user query.
 * If AI determines that all files are necessary, then if the
 * 'Use filter Phrase Document to answer broad query' checkbox is checked,
 * we use the filter phrase document as the context to answer the broad
 * query. When this checkbox is not checked, the context for the global
 * query is built as normal using retrieval/enrichment.
 * You may want to present the user with the answers from both routes.
 * **************************************************************************/
private async Task<ClsFilterResult> ApplyFilter()
{
    string dynamicSystemPrompt = "";
    LangChain.Providers.Message dynamicMessage;

    ClsFilterResult result = new ClsFilterResult();

    // ensure that all files have a phrase document
    if (IndexDoc == "")
    {
        filterTokenCount = 0;
        foreach (ClsFiles file in files)
        {
            if (file.PhraseDoc == "") file.PhraseDoc = file.ven.VesCreateDocumentIndex(filterTokenPct, out file.FilterTokens);

            if (IndexDoc.Length > 0) IndexDoc += "\n\n";
            IndexDoc += file.PhraseDoc;
            filterTokenCount += file.FilterTokens;
        }
    }
    foreach (ClsFiles file in files) file.InFilter = false;  // reset

    // to take full advantage of caching, we place the static content before the chat history
    dynamicSystemPrompt = Ven.VesGetFilterSystemPrompt() +
                 "\n\nUSE THIS DOCUMENT INDEX CONTEXT TO ANSWER THE USER'S QUESTION:\n" +
                 IndexDoc;

    // create a combined list: system message followed by the filter chat history
    dynamicMessage = new LangChain.Providers.Message(dynamicSystemPrompt, LangChain.Providers.MessageRole.System);

    filterChatHistory.Add(new LangChain.Providers.Message(newUserQuestion, LangChain.Providers.MessageRole.Human));
    List<LangChain.Providers.Message> payload = [dynamicMessage, .. filterChatHistory];  // .. appends all elements of the filterChatHistory list

    // When the filter is enabled, miniChat (Luna) is reserved for
    // only examining the user question against the collection-wide document index.
    // This ensures that we are not only charged the lower miniChat cost, but also that such calls 
    // are highly discounted due to an assured cache hit.
    string answer = await SendToAI(miniChat, payload, filterChatHistory, false);

    answer = answer.Trim().ToUpper();

    // Does AI think that no document is relevant, or that all documents are needed
    // to answer the user question?
    // If so, we follow the usual route of retrieval/enrichment over the entire
    // collection.
    if (answer == "NONE" || answer == "ALL") return result;
    else if (answer == "INDEX")
    {  // the answer is found in the index document, so build a payload using the index document
        if (!FldUsePhraseDocForBroadQuery.Checked) return result;   // The user does not want to use the index, go back to using full retrieval/enrichment
        // In your app, you might present the user the answers from both sources, indicating the source: file collection or index document

        dynamicSystemPrompt = Ven.VesGetIndexSystemPrompt() + IndexDoc;

        // skip adding the user prompt because it was already added to chatHistory at the beginning of the FldSend event handler

        // create a combined message and the payload           
        dynamicMessage = new LangChain.Providers.Message(dynamicSystemPrompt, LangChain.Providers.MessageRole.System);
        result.payload = [dynamicMessage, .. chatHistory];

        TokenMessage = $"Answered from collection index ({files.Count} documents). " + $"Number of index document tokens sent: {filterTokenCount}";

        result.FilterApplied = true;
        return result;
    }
    else
    {    // list of doc-id and confidence level, example:  TE582281:92|TE851927:70|TE642026:35   (expect spaces between the elements)
        // check if we have at least 70% confidence level
        int RequiredConfidenceLevel = 70;  // you can change this to any level needed by your application
        bool WeHaveConfidence = false;

        // extract doc-ids and confidence levels
        List<string> DocIds = new List<string>();
        string[] items = answer.Split('|');
        int NumItems = items.Length;
        for (int i = 0; i < NumItems; i++)
        {
            items[i] = items[i].Trim();
            string[] subitems = items[i].Split(":");
            if (subitems.Length == 2)
            {
                string id = subitems[0].Trim();
                if (string.IsNullOrEmpty(id)) continue;
                if (!int.TryParse(subitems[1].Trim(), out int lvl)) lvl = 0;
                if (lvl >= RequiredConfidenceLevel) WeHaveConfidence = true;
                DocIds.Add(id);
            }
        }

        if (!WeHaveConfidence) return result;  // can't use the filter, resort to the regular retrieval/enrichment

        int tokens = 0;
        if (filesDict == null) return result;  // this should not happen
        foreach (string DocId in DocIds)
        {
            if (filesDict.TryGetValue(DocId, out ClsFiles? file))
            {
                file.InFilter = true;
                tokens += file.TokenCount;
            }
        }

        if (tokens == 0) return result;  // no files found.  This should not happen

        // check if the token count of the filtered file set is less than the user allowance. In that case, send the entire context
        int AveTokenPerChunk = TotalTokenCount / TotalChunkCount;
        int RetrievalChunkLimit = TotalChunkCount * retrievalPct;
        int RetrievalTokenLimit = AveTokenPerChunk * RetrievalChunkLimit / 100;
        int TotalUserAllowance = RetrievalTokenLimit + TotalTokenCount * enrichmentPct / 100;  // the user is okay with sending up to this many tokens

        // Exceeded: go through the regular retrieval/enrichment over the filtered set.
        // The fewer files in the filter, the stronger the context sent to AI, because the token allowance gets spread over fewer files.
        // Filtration here provides strong context for the filtered files to send to AI.
        if (tokens > TotalUserAllowance)
        {
            result.FilterApplied = true;
            return result;
        }

        // Great success!  We can send each one of these files completely
        // and still stay within the user token allowance

        // get a single flattened collection of all chunks:
        string[] allChunksArray = files
            .Where(f => f.InFilter)
            .SelectMany(f => f.chunk)
            .ToArray();

        // combined together:
        string allChunksJoined = string.Join("\n\n", allChunksArray);

        dynamicSystemPrompt = Dan.DasGetSystemMessage() +
                             "\n\nUSE THIS EXTRACTED DOCUMENT CONTEXT TO ANSWER THE USER'S QUESTION:\n" + allChunksJoined;

        // No need to add the user prompt because it was already added to chatHistory at the beginning of the FldSend event handler

        // create a combined message and the payload           
        dynamicMessage = new LangChain.Providers.Message(dynamicSystemPrompt, LangChain.Providers.MessageRole.System);
        result.payload = [dynamicMessage, .. chatHistory];

        TokenMessage = $"Answered from filtered file collection ({DocIds.Count} documents). " + $"Number of document tokens sent: {tokens}";

    }


    return result;
}

Retrieval: LOCAL and GLOBAL strategies

One collection-wide search for pointed questions; for broad questions, a coverage share is spread across every file and the remainder is distributed in proportion to each file's hits, then each file is searched with its own quota.

/************************************************************************
 * GetRetrievalEnrichmentContext:
 * The user asked a broad question.  To get the content, we will:
 * 1. Do retrieval over the entire collection.
 * 2. Isolate the chunks in the retrieval for each file.
 *    For global query:
 *    2a. Calculate the proportion of each file in the hits.  Say file #2 gets 60% of the hits, file #4 gets the remaining chunks,
 *        and the other files are not represented in the retrieval search.
 *    2b. Find the chunk quota for each file this way:
 *        Distribute the 'coverage share' percentage of the total allowed retrieval chunks equally to all files
 *        whether they were found in the retrieval set or not.  This is what makes the search global.
 *        However, some files may not get any if there are not enough to share.
 *        The filtration routing step remedies this gap. The filtration reduces the number of files of 
 *        interest for the current query. 
 *    2c. Find the retrieved chunks for each file using the quota calculated in step 2b. 
 *        (steps 2a through 2c are performed in the DoGlobalDistribution method below)
 * 3. Do enrichment for each file with retrieval, thus expanding the chunks of interest.
 * 4. Accumulate the chunks of interest for each file in a common array.
 * 5. Add the system message and chat history to make the final payload.
 * ********************************************************************/
private async Task<List<LangChain.Providers.Message>>? GetRetrievalEnrichmentContext()
{
    var sw = StartStopwatch();

    // initialize the retrieved and enriched sequences before adding new retrieved sequences 
    foreach (ClsFiles file in files)
    {
        file.RetrieveSeq = null;
        file.EnrichedSeq = null;
        file.RetrievalTokenCount = 0;
        file.EnrichmentTokenCount = 0;
    }

    float[] queryVector = await GetQueryVector();

    int searchLimit = (TotalChunkCount * retrievalPct + 99) / 100;   // +99 to round up 
    if (searchLimit < 1) searchLimit = 1;

    var searchSettings = new VectorSearchSettings
    {
        NumberOfResults = searchLimit   // our search limit
    };

    IVectorCollection? CurVectorCollection = await GetVectorCollection(false);   // collection containing documents that are in filter

    LogTime(sw, "GetVectorCollection");

    VectorSearchResponse searchResponse =
        await CurVectorCollection.SearchAsync(queryVector, searchSettings);
    LogTime(sw, "SearchAsync");

    // Build RetrieveSeq for each file from the common searchResponse result
    // Create a dictionary lookup map: O(M) time complexity
    // Assumes DocId is unique across files. If not unique, use ToLookup() instead.
    TotalRetrievalTokens = 0;   // will be calculated below
    foreach (ClsFiles file in files) file.RetrievalTokenCount = 0;

    foreach (var item in searchResponse.Items)
    {
        if (item.Metadata == null) continue;

        // Safely extract metadata (avoids potential KeyNotFoundException)
        if (!item.Metadata.TryGetValue("ssDocId", out var docIdObj) ||
            !item.Metadata.TryGetValue("ssSeq", out var seqObj))
            continue;

        string docId = (string)docIdObj;
        int seq = Convert.ToInt32(seqObj);  

        // O(1) constant-time lookup instead of scanning the whole list
        if (filesDict.TryGetValue(docId, out ClsFiles? file))
        {
            // Null-coalescing assignment ensures initialization happens only once
            file.RetrieveSeq ??= new SortedSet<int>();
            file.RetrieveSeq.Add(seq);

            int SeqTokenCount = file.ven.VesGetChunkTokenCount(seq);

            TotalRetrievalTokens += SeqTokenCount;  // for the collection
            file.RetrievalTokenCount += SeqTokenCount;  // for this file
        }
    }
    LogTime(sw, "build retrieval seq");

    if (globalQuery) await DoGlobalDistribution();
    LogTime(sw, "DoGlobalDistribution");

    // do Vector Enrichment for the retrieved sequences for each file

    int QueryRetrievalTokens = 0;

    int QueryEnrichmentTokens = 0;
    foreach (ClsFiles file in files)
    {
        if (!file.InFilter || file.RetrieveSeq == null) continue;

        await DoVectorEnrichment(file);

        QueryRetrievalTokens += file.RetrievalTokenCount;
        QueryEnrichmentTokens += file.EnrichmentTokenCount;
    }
    LogTime(sw, "DoVectorEnrichment");

    // collect the chunks for the query
    StringBuilder sb = new StringBuilder();

    foreach (ClsFiles file in files)
    {
        if (file.chunk == null || file.RetrieveSeq == null || !file.InFilter) continue;

        if (sb.Length > 0) sb.Append("\n\n");

        if (file.EnrichedSeq != null)
        {
            if (file.EnrichedSeq.Count == file.chunk.Length) // all chunks in the enriched sequence
            {
                sb.Append(string.Join("\n\n", file.chunk));
            }
            else
            {
                // LINQ: Pulls only the chunks at the specified indices and joins them
                var selectedChunks = file.EnrichedSeq.Select(seq => file.chunk[seq]);
                sb.Append(string.Join("\n\n", selectedChunks));
            }
        }
        else
        {
            if (file.RetrieveSeq.Count == file.chunk.Length)   // all chunks in the retrieved sequence
            {
                sb.Append(string.Join("\n\n", file.chunk));
            }
            else
            {
                // LINQ: Pulls only the chunks at the specified indices and joins them
                var selectedChunks = file.RetrieveSeq.Select(seq => file.chunk[seq]);
                sb.Append(string.Join("\n\n", selectedChunks));
            }
        }
    }

    string chunks = sb.ToString();

    LogTime(sw, "collect chunks");

    // To take full advantage of caching, we place the static content before the chat history
    string dynamicSystemPrompt = Dan.DasGetSystemMessage() +
                 "\n\nUSE THIS EXTRACTED DOCUMENT CONTEXT TO ANSWER THE USER'S QUESTION:\n" +
                 chunks;

    // create a combined list: system message followed by the chat history
    LangChain.Providers.Message dynamicMessage = new LangChain.Providers.Message(dynamicSystemPrompt,
                                                                         LangChain.Providers.MessageRole.System);
    List<LangChain.Providers.Message> payload = [dynamicMessage, .. chatHistory];

    TokenMessage = $"Number of document tokens sent: {filterTokenCount + QueryRetrievalTokens + QueryEnrichmentTokens}";
    if (FldUseFilter.Checked) TokenMessage += $", Filter tokens: {filterTokenCount} (uses cache)";
    TokenMessage += $", Retrieval tokens: {QueryRetrievalTokens}, Enrichment tokens: {QueryEnrichmentTokens}";
    if (globalQuery) TokenMessage += ", Broad query.";
    else TokenMessage += ", Pointed query.";

    return payload;
}

/*****************************************************************************
 * DoGlobalDistribution:
 *    1. Calculate the proportion of each file in the hits.  Say file #2 gets 60% of the hits, file #4 gets the remaining chunks,
 *       and the other files are not represented in the retrieval search.
 *    2. Find the chunk quota for each file this way:
 *       Distribute the 'coverage share' percentage of the total allowed retrieval chunks equally to all files
 *       whether they were found in the retrieval set or not.  This is what makes the search global.
 *       However, some files may not get any if there are not enough to share.
 *       The filtration routing step remedies this gap. The filtration reduces the number of files of 
 *       interest for the current query. 
 *    3. Find the retrieved chunks for each file using the quota calculated in step 2. 
 ****************************************************************************/
private async Task DoGlobalDistribution()
{
    int TotalRetrievalQuota = TotalChunkCount * retrievalPct / 100;   // number of chunks for equal distribution among the files
    int TotalGlobalQuota = TotalRetrievalQuota * globalSpreadPct / 100;

    int FileCount = files.Count;
    int ActualGlobalQuota = 0;
    var sw = StartStopwatch();

    foreach (ClsFiles file in files)
    {
        file.quota = 0;
        if (!file.InFilter) continue;

        file.quota = TotalGlobalQuota / FileCount;
        if (file.quota < 1) file.quota = 1;
        ActualGlobalQuota += file.quota;
    }

    int RemainingQuota = TotalRetrievalQuota - ActualGlobalQuota;   // left over to spread over the files with hits

    // apportion the remaining quota to the files with hits
    foreach (ClsFiles file in files)
    {
        if (!file.InFilter) continue;

        if (file.RetrieveSeq != null)    // file.RetrieveSeq comes from the previous step, when we did the first vector search over the entire collection to find the files with hits
        {
            file.quota += (RemainingQuota * file.RetrieveSeq.Count) / TotalRetrievalQuota;
        }
        file.RetrievalTokenCount = 0;  // reinitialize for global query
    }

    // Calculate RetrieveSeq for each file using its quota
    TotalRetrievalTokens = 0;  // total retrieval for the collection

    float[] queryVector = await GetQueryVector();

    foreach (ClsFiles file in files)
    {
        if (!file.InFilter) continue;

        file.RetrieveSeq = new SortedSet<int>();

        var searchSettings = new VectorSearchSettings
        {
            NumberOfResults = file.quota   // get this many hits
        };

        VectorSearchResponse searchResponse =
            await file.vectorCollection.SearchAsync(queryVector, searchSettings);

        // Build RetrieveSeq for this file from the searchResponse result
        foreach (var item in searchResponse.Items)
        {
            if (item.Metadata == null) continue;  // shouldn't happen

            // Safely extract metadata (avoids potential KeyNotFoundException)
            if (!item.Metadata.TryGetValue("ssSeq", out var seqObj)) continue;

            int seq = (int)seqObj;
            file.RetrieveSeq.Add(seq);

            int SeqTokenCount = file.ven.VesGetChunkTokenCount(seq);
            file.RetrievalTokenCount += SeqTokenCount;  // for this file
        }
    }
    LogTime(sw, "do global inner");
}

/******************************************************************************
    GetQueryVector:
    Get the vector for the latest query
*******************************************************************************/
private async Task<float[]>? GetQueryVector()
{
    // This will return an array containing your single numerical coordinate array.
    float[][] embeddingBatch = await embeddingsProvider.CreateEmbeddingsAsync(new[] { newUserQuestion });

    // Extract the first (and only) vector element from the returned batch
    return embeddingBatch[0];
}

Enriching retrieved chunks with Ven

For each document with hits: VesBeginExpansion, VesAddRelatedChunks to complete tables, revisions and comments under a token budget, optional revision and comment additions, then VesGetExpandedSeq. The two classifiers are available when an application wants to gate revision and comment enrichment on the question.

/*****************************************************************************
 * DoVectorEnrichment:
 * Use the Vector Enrichment Library to add structurally related chunks to
 * the retrieved set of sequences.
******************************************************************************/
private async Task DoVectorEnrichment(ClsFiles file)
{
    // In this method, we are doing 'completion' enrichment if the document
    // has tables, revisions, or comments.
    // However, if you wish to perform other types of enrichment,
    // please remove this 'if' condition and use the various methods from the 
    // Vector Enrichment Library (vei package) to perform enrichment
    // suitable for your application.
    Stopwatch sw = StartStopwatch();

    if (!file.HasTables && !file.HasRevisions && !file.HasComments)
    {
        file.EnrichedSeq = file.RetrieveSeq;
        file.EnrichmentTokenCount = 0;  // no enrichment
        return;
    }

    bool CheckForRevisionCommentUsingAI = false;   // this could be an expensive call, about 900 ms for the first call
    if (CheckForRevisionCommentUsingAI)
    {
        if (file.HasRevisions && !UserQueryCheckedForRevisions)
        {
            QueryIsAboutRevisions = await IsQueryAboutRevisions(newUserQuestion);
            UserQueryCheckedForRevisions = true;
        }

        if (file.HasComments && !UserQueryCheckedForComments)
        {
            QueryIsAboutComments = await IsQueryAboutComments(newUserQuestion);
            UserQueryCheckedForComments = true;
        }
    }
    else QueryIsAboutRevisions = QueryIsAboutComments = true; // Enrichment Library calls are much more efficient,
                                                              // so let's enrich for revisions/comments if the file has them,
                                                              // without using AI to first check the query type as above

    LogTime(sw, "after AI query");

    if (TotalTokenCount == 0) return;

    int TotalEnrichmentBudget = TotalTokenCount * enrichmentPct / 100;  // number of tokens budgeted to add for structural enrichment
    float EnrichmentBudgetPerRetrievalToken = (float)TotalEnrichmentBudget / TotalRetrievalTokens;

    int EnrichmentBudget = (int)(EnrichmentBudgetPerRetrievalToken * file.RetrievalTokenCount);  // allocate the budget in proportion to the number of tokens retrieved for the file

    // The Vector Enrichment Library allows you to carry out enrichment at a granular level.
    // Your application can choose to add the enrichment relevant to you.
    // For the sake of simplicity in this demo, we will use the big-hammer method VesAddRelatedChunks, forsaking the granular control
    // offered by the other methods.
    // The Vector Enrichment Library includes two types of enrichment:
    // 1. Completion type:  This includes adding surrounding chunks so that tables, revisions, and comments are presented to AI
    //    in a structurally complete manner.  An incomplete table is worse than no table at all.
    // 2. Addition type: This type includes the addition of pages, and spread-out revisions and comments, to present comprehensive 
    //    information to AI.
    // Of the two above, the completion types are the most critical to incorporate.


    Object exp = null;   // Expansion object - the object that carries out enrichment expansion 
    try
    {
        exp = file.ven.VesBeginExpansion(file.RetrieveSeq);  // For each query, create the expansion object passing the zero-based retrieved sequences
    }
    catch (ArgumentException ex)
    {
        LblStatus.Text = $"Error creating expansion object: {ex.Message}";
    }
    LogTime(sw, "after creating exp object");

    if (file.ven != null && exp != null)
    {
        //int InitialTokenCount=file.ven.VesGetTokenCount(exp);  // example of finding the number of tokens in the retrieved sequences


        int TokensAdded = file.ven.VesAddRelatedChunks(exp,       // current expansion object
                                file.HasRevisions && QueryIsAboutRevisions,  //    True to include all contiguous chunks containing revisions, to complete the revisions for the reviewer authors
                                                                             //    referred to in the original set
                                file.HasComments && QueryIsAboutComments,    //    True to include all contiguous chunks containing comments, to complete the comments by the authors
                                                                             //    referred to in the original set

                     /*AllTables*/ false,       //    Set to 'true' to include all tables in the document.  
                                                //    If set to false, this method ensures that all tables in the original set and
                                                //    partial adjacent tables are included in the final set.
                                                //    Both options ensure no orphan or incomplete table is included in the final set.
                                                //    It is important that AI is given complete context to decipher 
                                                //    the contextual relationship between table columns, and the significance of 
                                                //    the spanned rows and columns. 
                                                //    This parameter is ignored if the document contains no tables.

               /*AdjacentTables*/   true,       //    When AllTables is false but AdjacentTables is true, the method not only completes
                                                //    the tables in the original set, but also completes adjacent tables found in the expanded chunks.
                                                //    This ensures that AI does not receive any incomplete tables.

                              EnrichmentBudget);//    Token Budget: try to limit the token count of the selected chunks to this limit. Internally, 
                                                //    when this limit is reached, this method 
                                                //    turns off all 'addition' such as adding pages or sections.  Only the 'completion' operations
                                                //    are performed, such as completing the selected table, revision, or comment chunks.
                                                //    A partial table confuses AI, and could be worse than no table at all.
                                                //    Pass a very large value for no limit (not recommended).

        LogTime(sw, $"tokens added: {TokensAdded}");

        file.EnrichmentTokenCount = TokensAdded;


        if (TokensAdded < EnrichmentBudget)
        {
            // If the question is about commented text, comment authors or document revisions
            if (file.HasComments && QueryIsAboutComments)
            {
                if (file.EnrichmentTokenCount < EnrichmentBudget) TokensAdded = file.ven.VesAddCommentChunks(exp, "");  // if the budget allows, do this addition
                file.EnrichmentTokenCount = TokensAdded;
            }

            if (file.HasRevisions && QueryIsAboutRevisions)
            {
                if (file.EnrichmentTokenCount < EnrichmentBudget) TokensAdded = file.ven.VesAddRevisionChunks(exp, "");
            }
        }

        file.EnrichmentTokenCount = TokensAdded; // TokensAdded returned from Ven methods is the cumulative enrichment tokens added

        //Ven.LogPrintf("initial/budget/added", InitialTokenCount, EnrichmentBudget,file.EnrichmentTokenCount);

        file.EnrichedSeq = file.ven.VesGetExpandedSeq(exp);  // get the expanded set of sequences

        file.ven.VesEndExpansion(exp);  // end the expansion

        LogTime(sw, "ves exp end, file: " + file.DocTitle);
    }
    else
    {
        file.EnrichedSeq = file.RetrieveSeq;
        file.EnrichmentTokenCount = 0;  // no enrichment
    }

    //LogTime(sw, "ves exp end ");

}

/******************************************************************************
    IsQueryAboutRevisions:
    Is the user querying about inserted and deleted text (redlining)?
*******************************************************************************/
private async Task<bool> IsQueryAboutRevisions(string userQuestion)
{
    try
    {
        // A quick, low-cost system prompt to categorize the question intent
        string routingPrompt = @"
       Analyze the user's question about a document. Categorize it into one of two strategies:
       - 'YES': The question is related to the revisions made by one or more reviewers to the document, such as 'What changes were made by Mary Hoffins?', 'Who made revisions to the document?'.
       - 'NO': The question is not about revisions made to the document, example: 'What is the warranty period?'.
       When in doubt, categorize as 'YES'.

       Respond with ONLY the word 'YES' or 'NO'.";

        var activeChat = miniChat;
        if (FldUseFilter.Checked) activeChat = fullChat;   // when using the filter, leave miniChat just for filtering so filtering can execute with discounted cache pricing

        ChatCompletion decision = await activeChat.CompleteChatAsync(
            new List<ChatMessage> {
               new SystemChatMessage(routingPrompt),
               new UserChatMessage(userQuestion)
            },
            new ChatCompletionOptions { MaxOutputTokenCount = 50 });
        return decision.Content[0].Text.Trim().ToUpperInvariant().StartsWith("YES");
    }
    catch (Exception)
    {
        LblStatus.Text = "Exception in IsQueryAboutRevisions";
        return false;  // in the unlikely case of a crash, assume the query is not about revisions
    }

}

/******************************************************************************
    IsQueryAboutComments:
    Is the user querying about document comments?
*******************************************************************************/
private async Task<bool> IsQueryAboutComments(string userQuestion)
{
    try
    {
        // A quick, low-cost system prompt to categorize the question intent
        string routingPrompt = @"
       Analyze the user's question about a document. Categorize it into one of two strategies:
       - 'YES': The question is related to the commented text and comment authors in this document, such as 'Which changes were made by John?', 'Which text did he comment on?', 'What were Mary's comments?'.
       - 'NO': The question is not about comments or commented text, example: 'What are the revenue items?'.
       When in doubt, categorize as 'YES'.

       Respond with ONLY the word 'YES' or 'NO'.";

        var activeChat = miniChat;
        if (FldUseFilter.Checked) activeChat = fullChat;   // when using the filter, leave miniChat just for filtering so filtering can execute with discounted cache pricing

        ChatCompletion decision = await activeChat.CompleteChatAsync(
            new List<ChatMessage> {
               new SystemChatMessage(routingPrompt),
               new UserChatMessage(userQuestion)
            },
            new ChatCompletionOptions { MaxOutputTokenCount = 50 });

        return decision.Content[0].Text.Trim().ToUpperInvariant().StartsWith("YES");
    }
    catch (Exception)
    {
        LblStatus.Text = "Exception in IsQueryAboutComments";
        return false;  // in the unlikely case of a crash, assume the query is not about comments
    }

}

Sending the payload to the model

The chat call uses the official OpenAI .NET SDK; LangChain messages are converted to SDK messages first. The system message from Dan.DasGetSystemMessage() plus the selected chunks is placed ahead of the chat history so the static part is served from cache.

/**********************************************************************
 * SendToAI:
 * Send the payload to AI
 * *******************************************************************/
private async Task<string> SendToAI(OpenAI.Chat.ChatClient activeChat, List<LangChain.Providers.Message> payload, List<LangChain.Providers.Message> history, bool UpdateUI)
{
    string responseText = "";

    try
    {
        // Using OpenAI for actual chat
        ChatCompletion completion = await activeChat.CompleteChatAsync(
            ConvertLangChainToOpenAiMessage(payload),
            new ChatCompletionOptions
            {
                MaxOutputTokenCount = 4000    // replaces MaxCompletionTokens = 1000; no Temperature
            });

        responseText = completion.Content[0].Text;

        // Track the text string in the history and update the HTML view
        history.Add(new LangChain.Providers.Message(responseText, LangChain.Providers.MessageRole.Ai));

        if (UpdateUI)
        {
            htmlResponse = PrevHtmlResponse;  // restore 
            UpdateChatBox(newUserQuestion, responseText);

            LblStatus.Text = TokenMessage;
            FldQuery.Text = ""; // clear for the next question
        }
    }
    catch (tryAGI.OpenAI.ApiException apiEx)
    {
        history.RemoveAt(history.Count - 1);  // Since we have an exception, meaning no AI response, undo the unanswered question as well from the history
        MessageBox.Show($"Error(sta-end): {apiEx.Message}\n\nBody: {apiEx.ResponseBody}");
    }
    catch (Exception ex)
    {
        history.RemoveAt(history.Count - 1);  // Since we have an exception, meaning no AI response, undo the unanswered question as well from the history
        MessageBox.Show($"Error(sta2-end): {ex.Message}");    // short message
    }

    return responseText;
}

/*********************************************************************
   ConvertLangChainToOpenAiMessage:
   Convert a LangChain message list to OpenAI SDK messages 
**********************************************************************/
static List<ChatMessage> ConvertLangChainToOpenAiMessage(IEnumerable<LangChain.Providers.Message> msgs)
{
    var list = new List<ChatMessage>();
    foreach (var m in msgs)
    {
        switch (m.Role)
        {
            case LangChain.Providers.MessageRole.System:
                list.Add(new SystemChatMessage(m.Content)); break;
            case LangChain.Providers.MessageRole.Ai:
                list.Add(new AssistantChatMessage(m.Content)); break;
            default:  // Human/user
                list.Add(new UserChatMessage(m.Content)); break;
        }
    }
    return list;
}

Documentation

The complete help for RAG Document Toolkit: license agreement, getting started, a step-by-step code example, and the full Dan and Ven API references.

using SubSystems.RagDocumentToolkit.Dan;
  using SubSystems.RagDocumentToolkit.Ven;
  

<PackageReference Include="dai" Version="1.0.0.6" />
  <PackageReference Include="vei" Version="1.0.0.5" />
  

<PackageReference Include="LangChain" Version="0.17.0" />
  <PackageReference Include="LangChain.Core" Version="0.17.0" />
  <PackageReference Include="LangChain.DocumentLoaders.Abstractions" Version="0.17.0" />
  <PackageReference Include="LangChain.Providers.OpenAI" Version="0.17.0" />
  <PackageReference Include="LangChain.Databases.InMemory" Version="0.17.0" />
  <PackageReference Include="LangChain.Databases.Sqlite" Version="0.17.0" />
  

int result = Dan.DasSetLicenseInfo("your-license-key", "your-license-number", "Your Company Name");
  

using SubSystems.RagDocumentToolkit.Dan;
  
  Dan dan = new Dan();
  dan.LogMsg   += LogDanMsg;    // optional: receive log messages
  dan.MdoNames += MdoNames;     // optional: supply names and terms for each chunk (see 1b)
  
  // import the file into the Dan object
  if (!dan.DasImportFile(NewFile, GetDocType(NewFile))) return false;
  
  // get the Markdown chunks and the metadata records (first page, last page; -1 = the entire document)
  Dan.DanResult mdo = dan.DasGetMarkdown(-1, -1);
  if (mdo == null) return false;
  
  // save these to your vector database
  file.DocId    = mdo.DocId;      // unique id for the document
  file.DocTitle = mdo.DocTitle;   // title found in the document, or the file path when there is none
  file.chunk    = mdo.chunks;     // string[]: the Markdown text of each chunk
  file.MetaRec  = mdo.MetaRecs;   // Dictionary<string,object>[]: one record per chunk
  

// PersonRx, OrgRx, Places, Terms and ContainsWord belong to your application
  Dan.ClsMdoNames MdoNames(object sender, string DocTitle, int ChunkIndex, string text)
  {
      var names   = new Dan.ClsMdoNames();
      var persons = new HashSet<string>();
      var orgs    = new HashSet<string>();
  
      // capture group 1 is the name without the honorific
      foreach (Match m in PersonRx.Matches(text)) persons.Add(m.Groups[1].Value.Trim());
  
      // capture group 1 is the full organization name
      foreach (Match m in OrgRx.Matches(text)) orgs.Add(m.Groups[1].Value.Trim());
  
      names.PersonNames    = persons.ToArray();
      names.OrgNames       = orgs.ToArray();
      names.PlaceNames     = Places.Where(p => ContainsWord(text, p)).ToArray();
      names.ImportantTerms = Terms.Where(t => ContainsWord(text, t)).ToArray();
  
      return names;
  }
  

using SubSystems.RagDocumentToolkit.Ven;
  
  // create the Vector Enrichment object from the document's complete metadata record set
  try
  {
      file.ven = new Ven(file.MetaRec);
  }
  catch (Exception ex)
  {
      LblStatus.Text = "Error creating Vector Enrichment object: " + ex.Message;
      return false;
  }
  
  // what does the document contain?
  file.HasTables    = file.ven.VesHasTables();
  file.HasRevisions = file.ven.VesHasRevisions();
  file.HasComments  = file.ven.VesHasComments();
  file.TokenCount   = file.ven.VesGetTokenCount();    // token count of the whole document
  
  // store each chunk with its metadata record in the vector database
  var docs = new List<Document>();
  for (int i = 0; i < file.chunk.Length; i++)
      docs.Add(new Document(content: file.chunk[i], metadata: file.MetaRec[i]));
  
  await vectorCollection.AddDocumentsAsync(embeddingsProvider, docs);
  
  files.Add(file);
  filesDict[file.DocId] = file;    // look up a file by its document id
  

// Build the index document once. Rebuild it when files are added or removed.
  if (IndexDoc == "")
  {
      foreach (ClsFiles file in files)
      {
          // index entry for one file, sized as a percentage of the file's tokens
          file.PhraseDoc = file.ven.VesCreateDocumentIndex(filterTokenPct, out file.FilterTokens);
  
          if (IndexDoc.Length > 0) IndexDoc += "\n\n";
          IndexDoc += file.PhraseDoc;
      }
  }
  
  // Static content goes first, so the index document is billed at the cached-input rate
  string systemPrompt = Ven.VesGetFilterSystemPrompt() +
                        "\n\nUSE THIS DOCUMENT INDEX CONTEXT TO ANSWER THE USER'S QUESTION:\n" +
                        IndexDoc;
  
  // AskModel is your own function. Use a low-cost model for this call.
  string answer = (await AskModel(miniChat, systemPrompt, newUserQuestion)).Trim().ToUpper();
  

foreach (ClsFiles f in files) f.InFilter = false;
  
  if (answer == "NONE" || answer == "ALL")
  {
      // no routing: regular retrieval and enrichment over the whole collection (steps 4 and 5)
  }
  else if (answer == "INDEX")
  {
      // answer the question from the index document itself
      systemPrompt = Ven.VesGetIndexSystemPrompt() + IndexDoc;
  }
  else
  {
      // example:  TE582281:92 | TE851927:70 | TE642026:35
      int  RequiredConfidenceLevel = 70;    // choose the level that suits your application
      bool confident = false;
      int  tokens = 0;
  
      foreach (string item in answer.Split('|'))
      {
          string[] parts = item.Split(':');
          if (parts.Length != 2 || !int.TryParse(parts[1].Trim(), out int level)) continue;
  
          if (level >= RequiredConfidenceLevel) confident = true;
  
          if (filesDict.TryGetValue(parts[0].Trim(), out ClsFiles? file))
          {
              file.InFilter = true;
              tokens += file.TokenCount;
          }
      }
  
      if (!confident || tokens == 0)
      {
          // no document stands out: regular retrieval and enrichment over the whole collection
      }
      else if (tokens <= TotalUserAllowance)
      {
          // the selected files fit within the token allowance: send them whole
          string context = string.Join("\n\n", files.Where(f => f.InFilter).SelectMany(f => f.chunk));
  
          systemPrompt = Dan.DasGetSystemMessage() +
                         "\n\nUSE THIS EXTRACTED DOCUMENT CONTEXT TO ANSWER THE USER'S QUESTION:\n" +
                         context;
      }
      else
      {
          // too large to send whole: run steps 4 and 5 over the selected files only,
          // treating the question as a broad query over this narrow set of files.
          // The token allowance is now spread over fewer files, so each gets stronger context.
          globalQuery = true;
      }
  }
  

VectorSearchResponse searchResponse = await vectorCollection.SearchAsync(queryVector, searchSettings);
  
  TotalRetrievalTokens = 0;
  foreach (var item in searchResponse.Items)
  {
      if (item.Metadata == null) continue;
  
      // ssDocId and ssSeq are written by Dan. A chunk without them did not come from Dan.
      if (!item.Metadata.TryGetValue("ssDocId", out var docIdObj) ||
          !item.Metadata.TryGetValue("ssSeq",   out var seqObj)) continue;
  
      if (filesDict.TryGetValue(Convert.ToString(docIdObj) ?? "", out ClsFiles? file))
      {
          int seq = Convert.ToInt32(seqObj);           // zero-based chunk sequence number
  
          file.RetrieveSeq ??= new SortedSet<int>();
          file.RetrieveSeq.Add(seq);
  
          int seqTokens = file.ven.VesGetChunkTokenCount(seq);
          file.RetrievalTokenCount += seqTokens;       // for this file
          TotalRetrievalTokens     += seqTokens;       // for the collection
      }
  }
  

// Share the enrichment budget among the files in proportion to their retrieved tokens
  int TotalEnrichmentBudget = TotalTokenCount * enrichmentPct / 100;
  int EnrichmentBudget = (int)((float)TotalEnrichmentBudget / TotalRetrievalTokens * file.RetrievalTokenCount);
  
  // Create an expansion object for this query from the retrieved sequence numbers
  object exp;
  try
  {
      exp = file.ven.VesBeginExpansion(file.RetrieveSeq);
  }
  catch (ArgumentException ex)
  {
      LblStatus.Text = "Error creating expansion object: " + ex.Message;
      file.EnrichedSeq = file.RetrieveSeq;    // use the chunks as retrieved
      return;
  }
  
  int TokensAdded = file.ven.VesAddRelatedChunks(exp,
          file.HasRevisions,    // complete each run of revisions found in the retrieved set
          file.HasComments,     // complete each run of comments found in the retrieved set
          false,                // AllTables: true = also add every other table, if the budget allows
          true,                 // AdjacentTables: also complete the tables met while expanding
          EnrichmentBudget);    // token budget for AllTables. Completions are always performed
  
  // If the budget allows, add comments and revisions found elsewhere in the document.
  // The second parameter selects an author. Pass "" for all authors.
  // Each method returns the cumulative number of enrichment tokens added.
  if (file.HasComments  && TokensAdded < EnrichmentBudget) TokensAdded = file.ven.VesAddCommentChunks(exp, "");
  if (file.HasRevisions && TokensAdded < EnrichmentBudget) TokensAdded = file.ven.VesAddRevisionChunks(exp, "");
  
  file.EnrichmentTokenCount = TokensAdded;
  file.EnrichedSeq = file.ven.VesGetExpandedSeq(exp);    // the expanded set of sequence numbers
  
  file.ven.VesEndExpansion(exp);
  

var sb = new StringBuilder();
  foreach (ClsFiles file in files)
  {
      if (!file.InFilter || file.RetrieveSeq == null) continue;
  
      var seqs = file.EnrichedSeq ?? file.RetrieveSeq;
  
      if (sb.Length > 0) sb.Append("\n\n");
      sb.Append(string.Join("\n\n", seqs.Select(seq => file.chunk[seq])));   // document order
  }
  
  // Static content goes before the chat history to take full advantage of caching
  string systemPrompt = Dan.DasGetSystemMessage() +
                        "\n\nUSE THIS EXTRACTED DOCUMENT CONTEXT TO ANSWER THE USER'S QUESTION:\n" +
                        sb.ToString();
  
  // send systemPrompt, the chat history and the new question to your main model