Semantic Deduplication

from dakera import DakeraClient
import time

client = DakeraClient(base_url="http://localhost:3300", api_key="dk-...")
AGENT = "research-bot"

# Ingest paper summaries (duplicates accumulate naturally)
papers = [
    ("BERT: Pre-training of Deep Bidirectional Transformers", 0.85),
    ("BERT language model pre-training for NLP", 0.82),  # near-dup of above
    ("Attention Is All You Need — transformer architecture", 0.90),
    ("The original transformer paper by Vaswani et al.", 0.88),  # near-dup of above
    ("GPT-3: Language Models are Few-Shot Learners", 0.87),
]

for content, importance in papers:
    client.store_memory(
        agent_id=AGENT,
        content=content,
        importance=importance,
        memory_type="semantic",
        tags=["paper", "nlp"]
    )

# Recall before dedup — duplicates consume slots
before = client.recall(agent_id=AGENT, query="transformer architecture NLP", top_k=5)
print(f"Before dedup: {len(before.memories)} results, likely 3-4 near-duplicates")

# Trigger deduplication
result = client.deduplicate(agent_id=AGENT)
print(f"Merged {result.merged} duplicates into {result.remaining} canonical memories")
print(f"Dedup elapsed: {result.elapsed_ms}ms")

# Recall after dedup — diverse results
after = client.recall(agent_id=AGENT, query="transformer architecture NLP", top_k=5)
print(f"After dedup: {len(after.memories)} results — diverse topics")
for mem in after.memories:
    print(f"  [{mem.importance:.2f}] {mem.content[:80]}")

# Schedule weekly dedup (example with a simple scheduler)
def run_weekly_dedup():
    while True:
        report = client.deduplicate(agent_id=AGENT)
        print(f"Weekly dedup: merged={report.merged}, remaining={report.remaining}")
        time.sleep(7 * 24 * 3600)  # 7 days

import { DakeraClient } from '@dakera-ai/dakera';

const client = new DakeraClient({ baseUrl: 'http://localhost:3300', apiKey: 'dk-...' });
const AGENT = 'research-bot';

// Ingest papers — near-duplicates accumulate from multiple sources
const papers = [
  { content: 'BERT: Pre-training of Deep Bidirectional Transformers', importance: 0.85 },
  { content: 'BERT language model pre-training for NLP tasks', importance: 0.82 }, // near-dup
  { content: 'Attention Is All You Need — transformer architecture paper', importance: 0.90 },
  { content: 'The original Vaswani et al. transformer paper 2017', importance: 0.88 }, // near-dup
  { content: 'GPT-3: Language Models are Few-Shot Learners', importance: 0.87 },
];

for (const paper of papers) {
  await client.storeMemory(AGENT, {
    content: paper.content,
    importance: paper.importance,
    memoryType: 'semantic',
    tags: ['paper', 'nlp']
  });
}

// Recall before — duplicates waste retrieval slots
const before = await client.recall(AGENT, 'transformer architecture NLP', { top_k: 5 });
console.log('Before:', before.memories.length, 'results (likely 3-4 near-duplicates)');

// Trigger semantic deduplication
const report = await client.deduplicate({ agentId: AGENT });
console.log(`Merged ${report.merged} duplicates, ${report.remaining} canonical memories remain`);
console.log(`Elapsed: ${report.elapsed_ms}ms`);

// Recall after — diverse, distinct papers
const after = await client.recall(AGENT, 'transformer architecture NLP', { top_k: 5 });
console.log('After:', after.memories.length, 'results — diverse topics per slot');
after.memories.forEach(m => console.log(`  [${m.importance}] ${m.content.slice(0, 80)}`));

// Schedule with setInterval (production: use cron or Dakera autopilot)
setInterval(async () => {
  const r = await client.deduplicate({ agentId: AGENT });
  console.log(`Scheduled dedup: merged=${r.merged}, remaining=${r.remaining}`);
}, 7 * 24 * 60 * 60 * 1000); // weekly

use dakera_rs::{Client, StoreMemoryRequest, RecallRequest};

let client = Client::new("http://localhost:3300", "dk-...");
let agent = "research-bot";

// Store paper summaries (some will be near-duplicates)
let papers = vec![
    ("BERT: Pre-training of Deep Bidirectional Transformers", 0.85f32),
    ("BERT language model for NLP pre-training", 0.82),  // near-dup
    ("Attention Is All You Need — transformer paper", 0.90),
];

for (content, importance) in papers {
    client.store_memory(agent, StoreMemoryRequest {
        content: content.into(),
        importance: Some(importance),
        memory_type: "semantic".into(),
        tags: vec!["paper".into(), "nlp".into()],
        ..Default::default()
    }).await?;
}

// Trigger deduplication via REST
// POST /v1/agents/research-bot/deduplicate
// Returns: { "merged": N, "remaining": M, "elapsed_ms": X }

// Verify recall diversity improved
let results = client.recall(agent, RecallRequest {
    query: "transformer architecture NLP".into(),
    top_k: Some(6),
    ..Default::default()
}).await?;

println!("After dedup: {} diverse results", results.memories.len());

package main

import (
    "context"
    "fmt"
    dakera "github.com/dakera-ai/dakera-go"
)

func main() {
    client := dakera.NewClient("http://localhost:3300", "dk-...")
    ctx := context.Background()
    agent := "research-bot"

    // Store papers with potential duplicates
    papers := []struct {
        Content    string
        Importance float64
    }{
        {"BERT: Pre-training of Deep Bidirectional Transformers", 0.85},
        {"BERT language model pre-training for NLP", 0.82}, // near-dup
        {"Attention Is All You Need — transformer architecture", 0.90},
    }

    for _, p := range papers {
        client.StoreMemory(ctx, agent, dakera.StoreMemoryRequest{
            Content:    p.Content,
            Importance: p.Importance,
            MemoryType: "semantic",
            Tags:       []string{"paper", "nlp"},
        })
    }

    // Trigger deduplication
    report, _ := client.Deduplicate(ctx, dakera.DeduplicateRequest{AgentID: agent})
    fmt.Printf("Dedup: merged=%d, remaining=%d, elapsed=%dms
",
        report.Merged, report.Remaining, report.ElapsedMs)

    // Verify diverse recall
    results, _ := client.Recall(ctx, agent, dakera.RecallRequest{
        Query: "transformer architecture NLP",
        TopK:  6,
    })
    fmt.Printf("After dedup: %d diverse results
", len(results.Memories))
}