Memory Compression

from dakera import DakeraClient
import openai
import uuid
from datetime import datetime, timedelta
from itertools import combinations
import numpy as np

client = DakeraClient(base_url="http://localhost:3300", api_key="dk-...")
oai = openai.OpenAI()
AGENT = "analytics-bot"

def cosine_sim(a, b):
    a, b = np.array(a), np.array(b)
    return float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)))

def cluster_memories(memories, threshold=0.78):
    """Greedy clustering by embedding similarity."""
    clusters = []
    assigned = set()
    for i, m in enumerate(memories):
        if i in assigned:
            continue
        cluster = [m]
        assigned.add(i)
        emb_i = m.embedding or []
        for j, n in enumerate(memories):
            if j in assigned or not n.embedding:
                continue
            if cosine_sim(emb_i, n.embedding) >= threshold:
                cluster.append(n)
                assigned.add(j)
        clusters.append(cluster)
    return clusters

def summarize_cluster(cluster):
    """LLM summarization of a memory cluster."""
    contents = "
".join([f"- {m.content}" for m in cluster])
    resp = oai.chat.completions.create(
        model="gpt-4o",
        messages=[{
            "role": "system",
            "content": "Summarize these analytics data points into a single dense paragraph. Preserve all quantitative values, dates, and trends. Highlight anomalies."
        }, {
            "role": "user",
            "content": f"Data points:
{contents}"
        }]
    )
    return resp.choices[0].message.content

def compress_agent_memories(date_cutoff_days: int = 30, min_cluster_size: int = 3):
    """Full compression pipeline for the analytics agent."""
    cutoff = datetime.now() - timedelta(days=date_cutoff_days)

    # Retrieve candidate memories (broad query to get many results)
    candidates = client.recall(
        agent_id=AGENT,
        query="daily metrics analytics report",
        min_importance=0.3,
        top_k=200
    )

    # Filter: exclude already-compressed and very recent memories
    raw = [m for m in candidates.memories
           if "compressed" not in (m.tags or [])
           and m.importance < 0.8]

    if len(raw) < min_cluster_size:
        print("Not enough memories to compress.")
        return

    # Cluster by semantic similarity
    clusters = cluster_memories(raw, threshold=0.78)
    print(f"Found {len(clusters)} clusters from {len(raw)} memories")

    compressed_count = 0
    for cluster in clusters:
        if len(cluster) < min_cluster_size:
            continue  # Skip tiny clusters — not worth compressing

        # Derive tags from cluster member tags
        all_tags = set()
        mem_ids = []
        for m in cluster:
            all_tags.update(m.tags or [])
            mem_ids.append(m.id)

        # Summarize cluster with LLM
        summary = summarize_cluster(cluster)
        if not summary or len(summary) < 50:
            continue  # Skip empty or trivial summaries

        cluster_id = str(uuid.uuid4())[:8]
        # Store compressed summary
        client.store_memory(
            agent_id=AGENT,
            content=summary,
            importance=0.90,
            memory_type="semantic",
            tags=["compressed", f"cluster-{cluster_id}"] + list(all_tags - {"compressed"})
        )

        # Hard-delete originals (analytics: no audit needed)
        client.batch_forget(request={
            "agent_id": AGENT,
            "memory_ids": mem_ids
        })
        compressed_count += len(cluster)
        print(f"Cluster {cluster_id}: {len(cluster)} memories -> 1 summary")

    print(f"Compression complete: {compressed_count} memories -> {len(clusters)} summaries")

# Run monthly compression
compress_agent_memories(date_cutoff_days=30, min_cluster_size=3)

import { DakeraClient } from '@dakera-ai/dakera';
import OpenAI from 'openai';
import { randomUUID } from 'crypto';

const client = new DakeraClient({ baseUrl: 'http://localhost:3300', apiKey: 'dk-...' });
const oai = new OpenAI();
const AGENT = 'analytics-bot';

function cosineSim(a: number[], b: number[]): number {
  const dot = a.reduce((sum, ai, i) => sum + ai * b[i], 0);
  const normA = Math.sqrt(a.reduce((s, x) => s + x * x, 0));
  const normB = Math.sqrt(b.reduce((s, x) => s + x * x, 0));
  return dot / (normA * normB);
}

function clusterMemories(memories: any[], threshold = 0.78): any[][] {
  const assigned = new Set<number>();
  const clusters: any[][] = [];
  for (let i = 0; i < memories.length; i++) {
    if (assigned.has(i)) continue;
    const cluster = [memories[i]];
    assigned.add(i);
    for (let j = i + 1; j < memories.length; j++) {
      if (assigned.has(j)) continue;
      const embI = memories[i].embedding ?? [];
      const embJ = memories[j].embedding ?? [];
      if (embI.length && embJ.length && cosineSim(embI, embJ) >= threshold) {
        cluster.push(memories[j]);
        assigned.add(j);
      }
    }
    clusters.push(cluster);
  }
  return clusters;
}

async function summarizeCluster(cluster: any[]): Promise<string> {
  const contents = cluster.map(m => `- ${m.content}`).join('
');
  const res = await oai.chat.completions.create({
    model: 'gpt-4o',
    messages: [
      { role: 'system', content: 'Summarize these analytics data points into a single dense paragraph. Preserve all quantitative values, dates, and anomalies.' },
      { role: 'user', content: `Data:
${contents}` }
    ]
  });
  return res.choices[0].message.content ?? '';
}

async function compressAgentMemories(dateCutoffDays = 30, minClusterSize = 3) {
  const candidates = await client.recall(AGENT, 'daily metrics analytics report', {
    min_importance: 0.3,
    top_k: 200
  });

  const raw = candidates.memories.filter(
    m => !(m.tags ?? []).includes('compressed') && m.importance < 0.8
  );

  if (raw.length < minClusterSize) { console.log('Not enough to compress.'); return; }

  const clusters = clusterMemories(raw);
  console.log(`Found ${clusters.length} clusters from ${raw.length} memories`);

  for (const cluster of clusters) {
    if (cluster.length < minClusterSize) continue;
    const summary = await summarizeCluster(cluster);
    if (!summary || summary.length < 50) continue;

    const allTags = [...new Set(cluster.flatMap(m => m.tags ?? []))].filter(t => t !== 'compressed');
    const clusterId = randomUUID().slice(0, 8);
    const memIds = cluster.map(m => m.id);

    await client.storeMemory(AGENT, {
      content: summary,
      importance: 0.90,
      memoryType: 'semantic',
      tags: ['compressed', `cluster-${clusterId}`, ...allTags]
    });
    await client.batchForget({ agent_id: AGENT, memory_ids: memIds });
    console.log(`Cluster ${clusterId}: ${cluster.length} memories -> 1 summary`);
  }
}

await compressAgentMemories(30, 3);

use dakera_rs::{Client, StoreMemoryRequest, RecallRequest};

let client = Client::new("http://localhost:3300", "dk-...");
let agent = "analytics-bot";

// 1. Retrieve candidate memories
let candidates = client.recall(agent, RecallRequest {
    query: "daily metrics analytics report".into(),
    min_importance: Some(0.3),
    top_k: Some(200),
    ..Default::default()
}).await?;

// 2. Filter: skip already-compressed and high-importance
let raw: Vec<_> = candidates.memories.iter()
    .filter(|m| !m.tags.contains(&"compressed".to_string()) && m.importance < 0.8)
    .collect();

// 3. Cluster by embedding similarity (your clustering logic)
let clusters = cluster_by_similarity(&raw, 0.78);

// 4. For each cluster: summarize with LLM and store
for cluster in clusters {
    if cluster.len() < 3 { continue; }
    let summary = summarize_with_llm(&cluster).await?; // your LLM call
    let cluster_id = uuid::Uuid::new_v4().to_string()[..8].to_string();
    let mem_ids: Vec<String> = cluster.iter().map(|m| m.id.clone()).collect();

    // Store compressed summary
    client.store_memory(agent, StoreMemoryRequest {
        content: summary,
        importance: Some(0.90),
        memory_type: "semantic".into(),
        tags: vec!["compressed".into(), format!("cluster-{}", cluster_id)],
        ..Default::default()
    }).await?;

    // Hard-delete originals via REST: POST /v1/memory/batch-forget
    // { "agent_id": "analytics-bot", "memory_ids": [mem_ids] }
    println!("Cluster {}: {} -> 1 summary", cluster_id, cluster.len());
}

package main

import (
    "context"
    "fmt"
    dakera "github.com/dakera-ai/dakera-go"
)

func compressAgentMemories(ctx context.Context, client *dakera.Client, agent string) {
    // 1. Retrieve candidate memories
    candidates, _ := client.Recall(ctx, agent, dakera.RecallRequest{
        Query: "daily metrics analytics report", MinImportance: 0.3, TopK: 200,
    })

    // 2. Filter: skip compressed and high-importance
    var raw []dakera.Memory
    for _, m := range candidates.Memories {
        if !contains(m.Tags, "compressed") && m.Importance < 0.8 {
            raw = append(raw, m)
        }
    }
    if len(raw) < 3 { fmt.Println("Not enough to compress."); return }

    // 3. Cluster by similarity (your clustering logic)
    clusters := clusterBySimilarity(raw, 0.78)
    fmt.Printf("Found %d clusters from %d memories
", len(clusters), len(raw))

    for _, cluster := range clusters {
        if len(cluster) < 3 { continue }
        summary := summarizeWithLLM(cluster) // your LLM call
        clusterID := randomID(8)
        memIDs := make([]string, len(cluster))
        for i, m := range cluster { memIDs[i] = m.ID }

        // Store compressed summary
        client.StoreMemory(ctx, agent, dakera.StoreMemoryRequest{
            Content:    summary,
            Importance: 0.90,
            MemoryType: "semantic",
            Tags:       []string{"compressed", "cluster-" + clusterID},
        })
        // Batch-delete originals
        client.BatchForget(ctx, dakera.BatchForgetRequest{AgentID: agent, MemoryIDs: memIDs})
        fmt.Printf("Cluster %s: %d memories -> 1 summary
", clusterID, len(cluster))
    }
}