{
  "schemaVersion": "1.0",
  "item": {
    "slug": "cut-your-tokens-97percent-savings-on-session-transcripts-via-observation-extraction",
    "name": "claw-compactor",
    "source": "tencent",
    "type": "skill",
    "category": "AI 智能",
    "sourceUrl": "https://clawhub.ai/aeromomo/cut-your-tokens-97percent-savings-on-session-transcripts-via-observation-extraction",
    "canonicalUrl": "https://clawhub.ai/aeromomo/cut-your-tokens-97percent-savings-on-session-transcripts-via-observation-extraction",
    "targetPlatform": "OpenClaw"
  },
  "install": {
    "downloadMode": "redirect",
    "downloadUrl": "/downloads/cut-your-tokens-97percent-savings-on-session-transcripts-via-observation-extraction",
    "sourceDownloadUrl": "https://wry-manatee-359.convex.site/api/v1/download?slug=cut-your-tokens-97percent-savings-on-session-transcripts-via-observation-extraction",
    "sourcePlatform": "tencent",
    "targetPlatform": "OpenClaw",
    "installMethod": "Manual import",
    "extraction": "Extract archive",
    "prerequisites": [
      "OpenClaw"
    ],
    "packageFormat": "ZIP package",
    "includedAssets": [
      "pyproject.toml",
      "README.md",
      "SKILL.md",
      "scripts/mem_compress.py",
      "scripts/estimate_tokens.py",
      "scripts/dedup_memory.py"
    ],
    "primaryDoc": "SKILL.md",
    "quickSetup": [
      "Download the package from Yavira.",
      "Extract the archive and review SKILL.md first.",
      "Import or place the package into your OpenClaw setup."
    ],
    "agentAssist": {
      "summary": "Hand the extracted package to your coding agent with a concrete install brief instead of figuring it out manually.",
      "steps": [
        "Download the package from Yavira.",
        "Extract it into a folder your agent can access.",
        "Paste one of the prompts below and point your agent at the extracted folder."
      ],
      "prompts": [
        {
          "label": "New install",
          "body": "I downloaded a skill package from Yavira. Read SKILL.md from the extracted folder and install it by following the included instructions. Then review README.md for any prerequisites, environment setup, or post-install checks. Tell me what you changed and call out any manual steps you could not complete."
        },
        {
          "label": "Upgrade existing",
          "body": "I downloaded an updated skill package from Yavira. Read SKILL.md from the extracted folder, compare it with my current installation, and upgrade it while preserving any custom configuration unless the package docs explicitly say otherwise. Then review README.md for any prerequisites, environment setup, or post-install checks. Summarize what changed and any follow-up checks I should run."
        }
      ]
    },
    "sourceHealth": {
      "source": "tencent",
      "status": "healthy",
      "reason": "direct_download_ok",
      "recommendedAction": "download",
      "checkedAt": "2026-04-30T16:55:25.780Z",
      "expiresAt": "2026-05-07T16:55:25.780Z",
      "httpStatus": 200,
      "finalUrl": "https://wry-manatee-359.convex.site/api/v1/download?slug=network",
      "contentType": "application/zip",
      "probeMethod": "head",
      "details": {
        "probeUrl": "https://wry-manatee-359.convex.site/api/v1/download?slug=network",
        "contentDisposition": "attachment; filename=\"network-1.0.0.zip\"",
        "redirectLocation": null,
        "bodySnippet": null
      },
      "scope": "source",
      "summary": "Source download looks usable.",
      "detail": "Yavira can redirect you to the upstream package for this source.",
      "primaryActionLabel": "Download for OpenClaw",
      "primaryActionHref": "/downloads/cut-your-tokens-97percent-savings-on-session-transcripts-via-observation-extraction"
    },
    "validation": {
      "installChecklist": [
        "Use the Yavira download entry.",
        "Review SKILL.md after the package is downloaded.",
        "Confirm the extracted package contains the expected setup assets."
      ],
      "postInstallChecks": [
        "Confirm the extracted package includes the expected docs or setup files.",
        "Validate the skill or prompts are available in your target agent workspace.",
        "Capture any manual follow-up steps the agent could not complete."
      ]
    },
    "downloadPageUrl": "https://openagent3.xyz/downloads/cut-your-tokens-97percent-savings-on-session-transcripts-via-observation-extraction",
    "agentPageUrl": "https://openagent3.xyz/skills/cut-your-tokens-97percent-savings-on-session-transcripts-via-observation-extraction/agent",
    "manifestUrl": "https://openagent3.xyz/skills/cut-your-tokens-97percent-savings-on-session-transcripts-via-observation-extraction/agent.json",
    "briefUrl": "https://openagent3.xyz/skills/cut-your-tokens-97percent-savings-on-session-transcripts-via-observation-extraction/agent.md"
  },
  "agentAssist": {
    "summary": "Hand the extracted package to your coding agent with a concrete install brief instead of figuring it out manually.",
    "steps": [
      "Download the package from Yavira.",
      "Extract it into a folder your agent can access.",
      "Paste one of the prompts below and point your agent at the extracted folder."
    ],
    "prompts": [
      {
        "label": "New install",
        "body": "I downloaded a skill package from Yavira. Read SKILL.md from the extracted folder and install it by following the included instructions. Then review README.md for any prerequisites, environment setup, or post-install checks. Tell me what you changed and call out any manual steps you could not complete."
      },
      {
        "label": "Upgrade existing",
        "body": "I downloaded an updated skill package from Yavira. Read SKILL.md from the extracted folder, compare it with my current installation, and upgrade it while preserving any custom configuration unless the package docs explicitly say otherwise. Then review README.md for any prerequisites, environment setup, or post-install checks. Summarize what changed and any follow-up checks I should run."
      }
    ]
  },
  "documentation": {
    "source": "clawhub",
    "primaryDoc": "SKILL.md",
    "sections": [
      {
        "title": "🦞 Claw Compactor",
        "body": "\"Cut your tokens. Keep your facts.\"\n\nCut your AI agent's token spend in half. One command compresses your entire workspace — memory files, session transcripts, sub-agent context — using 5 layered compression techniques. Deterministic. Mostly lossless. No LLM required."
      },
      {
        "title": "Features",
        "body": "5 compression layers working in sequence for maximum savings\nZero LLM cost — all compression is rule-based and deterministic\nLossless roundtrip for dictionary, RLE, and rule-based compression\n~97% savings on session transcripts via observation extraction\nTiered summaries (L0/L1/L2) for progressive context loading\nCJK-aware — full Chinese/Japanese/Korean support\nOne command (full) runs everything in optimal order"
      },
      {
        "title": "5 Compression Layers",
        "body": "#LayerMethodSavingsLossless?1Rule engineDedup lines, strip markdown filler, merge sections4-8%✅2Dictionary encodingAuto-learned codebook, $XX substitution4-5%✅3Observation compressionSession JSONL → structured summaries~97%❌*4RLE patternsPath shorthand ($WS), IP prefix, enum compaction1-2%✅5Compressed Context Protocolultra/medium/light abbreviation20-60%❌*\n\n*Lossy techniques preserve all facts and decisions; only verbose formatting is removed."
      },
      {
        "title": "Quick Start",
        "body": "git clone https://github.com/aeromomo/claw-compactor.git\ncd claw-compactor\n\n# See how much you'd save (non-destructive)\npython3 scripts/mem_compress.py /path/to/workspace benchmark\n\n# Compress everything\npython3 scripts/mem_compress.py /path/to/workspace full\n\nRequirements: Python 3.9+. Optional: pip install tiktoken for exact token counts (falls back to heuristic)."
      },
      {
        "title": "Architecture",
        "body": "┌─────────────────────────────────────────────────────────────┐\n│                      mem_compress.py                        │\n│                   (unified entry point)                     │\n└──────┬──────┬──────┬──────┬──────┬──────┬──────┬──────┬────┘\n       │      │      │      │      │      │      │      │\n       ▼      ▼      ▼      ▼      ▼      ▼      ▼      ▼\n  estimate compress  dict  dedup observe tiers  audit optimize\n       └──────┴──────┴──┬───┴──────┴──────┴──────┴──────┘\n                        ▼\n                  ┌────────────────┐\n                  │     lib/       │\n                  │ tokens.py      │ ← tiktoken or heuristic\n                  │ markdown.py    │ ← section parsing\n                  │ dedup.py       │ ← shingle hashing\n                  │ dictionary.py  │ ← codebook compression\n                  │ rle.py         │ ← path/IP/enum encoding\n                  │ tokenizer_     │\n                  │   optimizer.py │ ← format optimization\n                  │ config.py      │ ← JSON config\n                  │ exceptions.py  │ ← error types\n                  └────────────────┘"
      },
      {
        "title": "Commands",
        "body": "All commands: python3 scripts/mem_compress.py <workspace> <command> [options]\n\nCommandDescriptionTypical SavingsfullComplete pipeline (all steps in order)50%+ combinedbenchmarkDry-run performance report—compressRule-based compression4-8%dictDictionary encoding with auto-codebook4-5%observeSession transcript → observations~97%tiersGenerate L0/L1/L2 summaries88-95% on sub-agent loadsdedupCross-file duplicate detectionvariesestimateToken count report—auditWorkspace health check—optimizeTokenizer-level format fixes1-3%"
      },
      {
        "title": "Global Options",
        "body": "--json — Machine-readable JSON output\n--dry-run — Preview changes without writing\n--since YYYY-MM-DD — Filter sessions by date\n--auto-merge — Auto-merge duplicates (dedup)"
      },
      {
        "title": "Real-World Savings",
        "body": "Workspace StateTypical SavingsNotesSession transcripts (observe)~97%Megabytes of JSONL → concise observation MDVerbose/new workspace50-70%First run on unoptimized workspaceRegular maintenance10-20%Weekly runs on active workspaceAlready-optimized3-12%Diminishing returns — workspace is clean"
      },
      {
        "title": "cacheRetention — Complementary Optimization",
        "body": "Before compression runs, enable prompt caching for a 90% discount on cached tokens:\n\n{\n  \"models\": {\n    \"model-name\": {\n      \"cacheRetention\": \"long\"\n    }\n  }\n}\n\nCompression reduces token count, caching reduces cost-per-token. Together: 50% compression + 90% cache discount = 95% effective cost reduction."
      },
      {
        "title": "Heartbeat Automation",
        "body": "Run weekly or on heartbeat:\n\n## Memory Maintenance (weekly)\n- python3 skills/claw-compactor/scripts/mem_compress.py <workspace> benchmark\n- If savings > 5%: run full pipeline\n- If pending transcripts: run observe\n\nCron example:\n\n0 3 * * 0 cd /path/to/skills/claw-compactor && python3 scripts/mem_compress.py /path/to/workspace full"
      },
      {
        "title": "Configuration",
        "body": "Optional claw-compactor-config.json in workspace root:\n\n{\n  \"chars_per_token\": 4,\n  \"level0_max_tokens\": 200,\n  \"level1_max_tokens\": 500,\n  \"dedup_similarity_threshold\": 0.6,\n  \"dedup_shingle_size\": 3\n}\n\nAll fields optional — sensible defaults are used when absent."
      },
      {
        "title": "Artifacts",
        "body": "FilePurposememory/.codebook.jsonDictionary codebook (must travel with memory files)memory/.observed-sessions.jsonTracks processed transcriptsmemory/observations/Compressed session summariesmemory/MEMORY-L0.mdLevel 0 summary (~200 tokens)"
      },
      {
        "title": "FAQ",
        "body": "Q: Will compression lose my data?\nA: Rule engine, dictionary, RLE, and tokenizer optimization are fully lossless. Observation compression and CCP are lossy but preserve all facts and decisions.\n\nQ: How does dictionary decompression work?\nA: decompress_text(text, codebook) expands all $XX codes back. The codebook JSON must be present.\n\nQ: Can I run individual steps?\nA: Yes. Every command is independent: compress, dict, observe, tiers, dedup, optimize.\n\nQ: What if tiktoken isn't installed?\nA: Falls back to a CJK-aware heuristic (chars÷4). Results are ~90% accurate.\n\nQ: Does it handle Chinese/Japanese/Unicode?\nA: Yes. Full CJK support including character-aware token estimation and Chinese punctuation normalization."
      },
      {
        "title": "Troubleshooting",
        "body": "FileNotFoundError on workspace: Ensure path points to workspace root (contains memory/ or MEMORY.md)\nDictionary decompression fails: Check memory/.codebook.json exists and is valid JSON\nZero savings on benchmark: Workspace is already optimized — nothing to do\nobserve finds no transcripts: Check sessions directory for .jsonl files\nToken count seems wrong: Install tiktoken: pip3 install tiktoken"
      },
      {
        "title": "Credits",
        "body": "Inspired by claude-mem by thedotmack\nBuilt by Bot777 🤖 for OpenClaw"
      },
      {
        "title": "License",
        "body": "MIT"
      }
    ],
    "body": "🦞 Claw Compactor\n\n\"Cut your tokens. Keep your facts.\"\n\nCut your AI agent's token spend in half. One command compresses your entire workspace — memory files, session transcripts, sub-agent context — using 5 layered compression techniques. Deterministic. Mostly lossless. No LLM required.\n\nFeatures\n5 compression layers working in sequence for maximum savings\nZero LLM cost — all compression is rule-based and deterministic\nLossless roundtrip for dictionary, RLE, and rule-based compression\n~97% savings on session transcripts via observation extraction\nTiered summaries (L0/L1/L2) for progressive context loading\nCJK-aware — full Chinese/Japanese/Korean support\nOne command (full) runs everything in optimal order\n5 Compression Layers\n#\tLayer\tMethod\tSavings\tLossless?\n1\tRule engine\tDedup lines, strip markdown filler, merge sections\t4-8%\t✅\n2\tDictionary encoding\tAuto-learned codebook, $XX substitution\t4-5%\t✅\n3\tObservation compression\tSession JSONL → structured summaries\t~97%\t❌*\n4\tRLE patterns\tPath shorthand ($WS), IP prefix, enum compaction\t1-2%\t✅\n5\tCompressed Context Protocol\tultra/medium/light abbreviation\t20-60%\t❌*\n\n*Lossy techniques preserve all facts and decisions; only verbose formatting is removed.\n\nQuick Start\ngit clone https://github.com/aeromomo/claw-compactor.git\ncd claw-compactor\n\n# See how much you'd save (non-destructive)\npython3 scripts/mem_compress.py /path/to/workspace benchmark\n\n# Compress everything\npython3 scripts/mem_compress.py /path/to/workspace full\n\n\nRequirements: Python 3.9+. Optional: pip install tiktoken for exact token counts (falls back to heuristic).\n\nArchitecture\n┌─────────────────────────────────────────────────────────────┐\n│                      mem_compress.py                        │\n│                   (unified entry point)                     │\n└──────┬──────┬──────┬──────┬──────┬──────┬──────┬──────┬────┘\n       │      │      │      │      │      │      │      │\n       ▼      ▼      ▼      ▼      ▼      ▼      ▼      ▼\n  estimate compress  dict  dedup observe tiers  audit optimize\n       └──────┴──────┴──┬───┴──────┴──────┴──────┴──────┘\n                        ▼\n                  ┌────────────────┐\n                  │     lib/       │\n                  │ tokens.py      │ ← tiktoken or heuristic\n                  │ markdown.py    │ ← section parsing\n                  │ dedup.py       │ ← shingle hashing\n                  │ dictionary.py  │ ← codebook compression\n                  │ rle.py         │ ← path/IP/enum encoding\n                  │ tokenizer_     │\n                  │   optimizer.py │ ← format optimization\n                  │ config.py      │ ← JSON config\n                  │ exceptions.py  │ ← error types\n                  └────────────────┘\n\nCommands\n\nAll commands: python3 scripts/mem_compress.py <workspace> <command> [options]\n\nCommand\tDescription\tTypical Savings\nfull\tComplete pipeline (all steps in order)\t50%+ combined\nbenchmark\tDry-run performance report\t—\ncompress\tRule-based compression\t4-8%\ndict\tDictionary encoding with auto-codebook\t4-5%\nobserve\tSession transcript → observations\t~97%\ntiers\tGenerate L0/L1/L2 summaries\t88-95% on sub-agent loads\ndedup\tCross-file duplicate detection\tvaries\nestimate\tToken count report\t—\naudit\tWorkspace health check\t—\noptimize\tTokenizer-level format fixes\t1-3%\nGlobal Options\n--json — Machine-readable JSON output\n--dry-run — Preview changes without writing\n--since YYYY-MM-DD — Filter sessions by date\n--auto-merge — Auto-merge duplicates (dedup)\nReal-World Savings\nWorkspace State\tTypical Savings\tNotes\nSession transcripts (observe)\t~97%\tMegabytes of JSONL → concise observation MD\nVerbose/new workspace\t50-70%\tFirst run on unoptimized workspace\nRegular maintenance\t10-20%\tWeekly runs on active workspace\nAlready-optimized\t3-12%\tDiminishing returns — workspace is clean\ncacheRetention — Complementary Optimization\n\nBefore compression runs, enable prompt caching for a 90% discount on cached tokens:\n\n{\n  \"models\": {\n    \"model-name\": {\n      \"cacheRetention\": \"long\"\n    }\n  }\n}\n\n\nCompression reduces token count, caching reduces cost-per-token. Together: 50% compression + 90% cache discount = 95% effective cost reduction.\n\nHeartbeat Automation\n\nRun weekly or on heartbeat:\n\n## Memory Maintenance (weekly)\n- python3 skills/claw-compactor/scripts/mem_compress.py <workspace> benchmark\n- If savings > 5%: run full pipeline\n- If pending transcripts: run observe\n\n\nCron example:\n\n0 3 * * 0 cd /path/to/skills/claw-compactor && python3 scripts/mem_compress.py /path/to/workspace full\n\nConfiguration\n\nOptional claw-compactor-config.json in workspace root:\n\n{\n  \"chars_per_token\": 4,\n  \"level0_max_tokens\": 200,\n  \"level1_max_tokens\": 500,\n  \"dedup_similarity_threshold\": 0.6,\n  \"dedup_shingle_size\": 3\n}\n\n\nAll fields optional — sensible defaults are used when absent.\n\nArtifacts\nFile\tPurpose\nmemory/.codebook.json\tDictionary codebook (must travel with memory files)\nmemory/.observed-sessions.json\tTracks processed transcripts\nmemory/observations/\tCompressed session summaries\nmemory/MEMORY-L0.md\tLevel 0 summary (~200 tokens)\nFAQ\n\nQ: Will compression lose my data? A: Rule engine, dictionary, RLE, and tokenizer optimization are fully lossless. Observation compression and CCP are lossy but preserve all facts and decisions.\n\nQ: How does dictionary decompression work? A: decompress_text(text, codebook) expands all $XX codes back. The codebook JSON must be present.\n\nQ: Can I run individual steps? A: Yes. Every command is independent: compress, dict, observe, tiers, dedup, optimize.\n\nQ: What if tiktoken isn't installed? A: Falls back to a CJK-aware heuristic (chars÷4). Results are ~90% accurate.\n\nQ: Does it handle Chinese/Japanese/Unicode? A: Yes. Full CJK support including character-aware token estimation and Chinese punctuation normalization.\n\nTroubleshooting\nFileNotFoundError on workspace: Ensure path points to workspace root (contains memory/ or MEMORY.md)\nDictionary decompression fails: Check memory/.codebook.json exists and is valid JSON\nZero savings on benchmark: Workspace is already optimized — nothing to do\nobserve finds no transcripts: Check sessions directory for .jsonl files\nToken count seems wrong: Install tiktoken: pip3 install tiktoken\nCredits\nInspired by claude-mem by thedotmack\nBuilt by Bot777 🤖 for OpenClaw\nLicense\n\nMIT"
  },
  "trust": {
    "sourceLabel": "tencent",
    "provenanceUrl": "https://clawhub.ai/aeromomo/cut-your-tokens-97percent-savings-on-session-transcripts-via-observation-extraction",
    "publisherUrl": "https://clawhub.ai/aeromomo/cut-your-tokens-97percent-savings-on-session-transcripts-via-observation-extraction",
    "owner": "aeromomo",
    "version": "6.0.0",
    "license": null,
    "verificationStatus": "Indexed source record"
  },
  "links": {
    "detailUrl": "https://openagent3.xyz/skills/cut-your-tokens-97percent-savings-on-session-transcripts-via-observation-extraction",
    "downloadUrl": "https://openagent3.xyz/downloads/cut-your-tokens-97percent-savings-on-session-transcripts-via-observation-extraction",
    "agentUrl": "https://openagent3.xyz/skills/cut-your-tokens-97percent-savings-on-session-transcripts-via-observation-extraction/agent",
    "manifestUrl": "https://openagent3.xyz/skills/cut-your-tokens-97percent-savings-on-session-transcripts-via-observation-extraction/agent.json",
    "briefUrl": "https://openagent3.xyz/skills/cut-your-tokens-97percent-savings-on-session-transcripts-via-observation-extraction/agent.md"
  }
}