{
  "schemaVersion": "1.0",
  "item": {
    "slug": "encoding-formats",
    "name": "Encoding & Formats",
    "source": "tencent",
    "type": "skill",
    "category": "通讯协作",
    "sourceUrl": "https://clawhub.ai/gitgoodordietrying/encoding-formats",
    "canonicalUrl": "https://clawhub.ai/gitgoodordietrying/encoding-formats",
    "targetPlatform": "OpenClaw"
  },
  "install": {
    "downloadMode": "redirect",
    "downloadUrl": "/downloads/encoding-formats",
    "sourceDownloadUrl": "https://wry-manatee-359.convex.site/api/v1/download?slug=encoding-formats",
    "sourcePlatform": "tencent",
    "targetPlatform": "OpenClaw",
    "installMethod": "Manual import",
    "extraction": "Extract archive",
    "prerequisites": [
      "OpenClaw"
    ],
    "packageFormat": "ZIP package",
    "includedAssets": [
      "SKILL.md"
    ],
    "primaryDoc": "SKILL.md",
    "quickSetup": [
      "Download the package from Yavira.",
      "Extract the archive and review SKILL.md first.",
      "Import or place the package into your OpenClaw setup."
    ],
    "agentAssist": {
      "summary": "Hand the extracted package to your coding agent with a concrete install brief instead of figuring it out manually.",
      "steps": [
        "Download the package from Yavira.",
        "Extract it into a folder your agent can access.",
        "Paste one of the prompts below and point your agent at the extracted folder."
      ],
      "prompts": [
        {
          "label": "New install",
          "body": "I downloaded a skill package from Yavira. Read SKILL.md from the extracted folder and install it by following the included instructions. Tell me what you changed and call out any manual steps you could not complete."
        },
        {
          "label": "Upgrade existing",
          "body": "I downloaded an updated skill package from Yavira. Read SKILL.md from the extracted folder, compare it with my current installation, and upgrade it while preserving any custom configuration unless the package docs explicitly say otherwise. Summarize what changed and any follow-up checks I should run."
        }
      ]
    },
    "sourceHealth": {
      "source": "tencent",
      "slug": "encoding-formats",
      "status": "healthy",
      "reason": "direct_download_ok",
      "recommendedAction": "download",
      "checkedAt": "2026-04-30T00:29:31.011Z",
      "expiresAt": "2026-05-07T00:29:31.011Z",
      "httpStatus": 200,
      "finalUrl": "https://wry-manatee-359.convex.site/api/v1/download?slug=encoding-formats",
      "contentType": "application/zip",
      "probeMethod": "head",
      "details": {
        "probeUrl": "https://wry-manatee-359.convex.site/api/v1/download?slug=encoding-formats",
        "contentDisposition": "attachment; filename=\"encoding-formats-1.0.0.zip\"",
        "redirectLocation": null,
        "bodySnippet": null,
        "slug": "encoding-formats"
      },
      "scope": "item",
      "summary": "Item download looks usable.",
      "detail": "Yavira can redirect you to the upstream package for this item.",
      "primaryActionLabel": "Download for OpenClaw",
      "primaryActionHref": "/downloads/encoding-formats"
    },
    "validation": {
      "installChecklist": [
        "Use the Yavira download entry.",
        "Review SKILL.md after the package is downloaded.",
        "Confirm the extracted package contains the expected setup assets."
      ],
      "postInstallChecks": [
        "Confirm the extracted package includes the expected docs or setup files.",
        "Validate the skill or prompts are available in your target agent workspace.",
        "Capture any manual follow-up steps the agent could not complete."
      ]
    },
    "downloadPageUrl": "https://openagent3.xyz/downloads/encoding-formats",
    "agentPageUrl": "https://openagent3.xyz/skills/encoding-formats/agent",
    "manifestUrl": "https://openagent3.xyz/skills/encoding-formats/agent.json",
    "briefUrl": "https://openagent3.xyz/skills/encoding-formats/agent.md"
  },
  "agentAssist": {
    "summary": "Hand the extracted package to your coding agent with a concrete install brief instead of figuring it out manually.",
    "steps": [
      "Download the package from Yavira.",
      "Extract it into a folder your agent can access.",
      "Paste one of the prompts below and point your agent at the extracted folder."
    ],
    "prompts": [
      {
        "label": "New install",
        "body": "I downloaded a skill package from Yavira. Read SKILL.md from the extracted folder and install it by following the included instructions. Tell me what you changed and call out any manual steps you could not complete."
      },
      {
        "label": "Upgrade existing",
        "body": "I downloaded an updated skill package from Yavira. Read SKILL.md from the extracted folder, compare it with my current installation, and upgrade it while preserving any custom configuration unless the package docs explicitly say otherwise. Summarize what changed and any follow-up checks I should run."
      }
    ]
  },
  "documentation": {
    "source": "clawhub",
    "primaryDoc": "SKILL.md",
    "sections": [
      {
        "title": "Encoding & Formats",
        "body": "Encode, decode, and inspect data in common formats. Covers Base64, URL encoding, hex, Unicode, JWTs, hashing, checksums, and serialization formats."
      },
      {
        "title": "When to Use",
        "body": "Decoding a Base64 string from an API response or config\nURL-encoding parameters for HTTP requests\nInspecting hex dumps of binary data\nDecoding JWT tokens to see claims\nComputing or verifying file checksums\nConverting between character encodings (UTF-8, Latin-1, etc.)\nUnderstanding wire formats (protobuf, MessagePack)"
      },
      {
        "title": "Encode and decode",
        "body": "# Encode string\necho -n \"Hello, World!\" | base64\n# SGVsbG8sIFdvcmxkIQ==\n\n# Decode string\necho \"SGVsbG8sIFdvcmxkIQ==\" | base64 -d\n# Hello, World!\n\n# Encode a file\nbase64 image.png > image.b64\ncat file.bin | base64\n\n# Decode a file\nbase64 -d image.b64 > image.png\n\n# Base64url (URL-safe variant: + → -, / → _, no padding)\necho -n \"Hello\" | base64 | tr '+/' '-_' | tr -d '='\n# Base64url decode\necho \"SGVsbG8\" | tr '-_' '+/' | base64 -d"
      },
      {
        "title": "In code",
        "body": "// JavaScript (browser + Node.js 16+)\nbtoa('Hello');                    // \"SGVsbG8=\"\natob('SGVsbG8=');                 // \"Hello\"\n\n// Node.js Buffer\nBuffer.from('Hello').toString('base64');           // \"SGVsbG8=\"\nBuffer.from('SGVsbG8=', 'base64').toString();      // \"Hello\"\n\n// Binary data\nBuffer.from(binaryData).toString('base64');\nBuffer.from(b64String, 'base64');\n\n# Python\nimport base64\n\nbase64.b64encode(b\"Hello\").decode()     # \"SGVsbG8=\"\nbase64.b64decode(\"SGVsbG8=\")            # b\"Hello\"\n\n# URL-safe Base64\nbase64.urlsafe_b64encode(b\"Hello\").decode()\nbase64.urlsafe_b64decode(\"SGVsbG8=\")"
      },
      {
        "title": "Encode and decode",
        "body": "# Python one-liner\npython3 -c \"from urllib.parse import quote; print(quote('hello world & foo=bar'))\"\n# hello%20world%20%26%20foo%3Dbar\n\n# Decode\npython3 -c \"from urllib.parse import unquote; print(unquote('hello%20world%20%26%20foo%3Dbar'))\"\n# hello world & foo=bar\n\n# curl does it automatically for --data-urlencode\ncurl -G --data-urlencode \"q=hello world & more\" https://api.example.com/search"
      },
      {
        "title": "In code",
        "body": "// JavaScript\nencodeURIComponent('hello world & foo=bar');\n// \"hello%20world%20%26%20foo%3Dbar\"\n\ndecodeURIComponent('hello%20world%20%26%20foo%3Dbar');\n// \"hello world & foo=bar\"\n\n// encodeURI vs encodeURIComponent:\nencodeURI('https://example.com/path?q=hello world');\n// \"https://example.com/path?q=hello%20world\" (preserves URL structure)\nencodeURIComponent('https://example.com/path?q=hello world');\n// \"https%3A%2F%2Fexample.com%2Fpath%3Fq%3Dhello%20world\" (encodes everything)\n\nfrom urllib.parse import quote, unquote, urlencode\n\nquote('hello world')            # 'hello%20world'\nunquote('hello%20world')        # 'hello world'\nurlencode({'q': 'hello world', 'page': 1})  # 'q=hello+world&page=1'"
      },
      {
        "title": "View and convert",
        "body": "# File hex dump\nxxd file.bin | head -20\nxxd -l 64 file.bin          # First 64 bytes only\n\n# Hex dump (compact, no ASCII)\nxxd -p file.bin\n\n# Convert hex to binary\necho \"48656c6c6f\" | xxd -r -p\n# Hello\n\n# od (alternative)\nod -A x -t x1z file.bin | head -20\n\n# hexdump\nhexdump -C file.bin | head -20\n\n# Python\npython3 -c \"print(bytes.fromhex('48656c6c6f').decode())\"  # Hello\npython3 -c \"print('Hello'.encode().hex())\"                 # 48656c6c6f"
      },
      {
        "title": "In code",
        "body": "// JavaScript\nBuffer.from('Hello').toString('hex');           // \"48656c6c6f\"\nBuffer.from('48656c6c6f', 'hex').toString();    // \"Hello\"\n\n// Number to hex\n(255).toString(16);     // \"ff\"\nparseInt('ff', 16);     // 255\n\n# Python\n\"Hello\".encode().hex()                  # '48656c6c6f'\nbytes.fromhex('48656c6c6f').decode()    # 'Hello'\nhex(255)                                # '0xff'\nint('ff', 16)                           # 255"
      },
      {
        "title": "Inspect characters",
        "body": "# Show Unicode code points\necho -n \"Hello 世界\" | python3 -c \"\nimport sys\nfor char in sys.stdin.read():\n    print(f'U+{ord(char):04X}  {char}  {char.encode(\\\"utf-8\\\").hex()}')\"\n# U+0048  H  48\n# U+0065  e  65\n# ...\n# U+4E16  世  e4b896\n# U+754C  界  e7958c\n\n# Convert Unicode escape to character\nprintf '\\u0048\\u0065\\u006c\\u006c\\u006f'   # Hello\necho -e '\\xE4\\xB8\\x96\\xE7\\x95\\x8C'       # 世界\n\n# File encoding detection\nfile -bi document.txt\n# text/plain; charset=utf-8"
      },
      {
        "title": "Encoding conversion",
        "body": "# Convert between encodings\niconv -f ISO-8859-1 -t UTF-8 input.txt > output.txt\niconv -f UTF-16 -t UTF-8 input.txt > output.txt\n\n# List available encodings\niconv -l\n\n# Python\npython3 -c \"\nwith open('latin1.txt', 'r', encoding='iso-8859-1') as f:\n    content = f.read()\nwith open('utf8.txt', 'w', encoding='utf-8') as f:\n    f.write(content)\n\""
      },
      {
        "title": "Common Unicode issues",
        "body": "BOM (Byte Order Mark):\n  UTF-8 BOM: EF BB BF at start of file\n  Remove: sed -i '1s/^\\xEF\\xBB\\xBF//' file.txt\n\nNormalization (NFC vs NFD):\n  \"é\" can be U+00E9 (one char) or U+0065 U+0301 (e + combining accent)\n  Python: import unicodedata; unicodedata.normalize('NFC', text)\n\nMojibake (wrong encoding):\n  \"café\" appears as \"café\" → file is UTF-8 but read as Latin-1\n  Fix: re-read with correct encoding"
      },
      {
        "title": "Decode a JWT",
        "body": "# JWT has 3 parts separated by dots: header.payload.signature\n# Each part is Base64url-encoded\n\n# Decode header and payload\nTOKEN=\"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c\"\n\n# Decode header\necho \"$TOKEN\" | cut -d. -f1 | tr '-_' '+/' | base64 -d 2>/dev/null | jq\n# {\"alg\":\"HS256\",\"typ\":\"JWT\"}\n\n# Decode payload\necho \"$TOKEN\" | cut -d. -f2 | tr '-_' '+/' | base64 -d 2>/dev/null | jq\n# {\"sub\":\"1234567890\",\"name\":\"John Doe\",\"iat\":1516239022}\n\n# One-liner function\njwt_decode() {\n    echo \"$1\" | cut -d. -f2 | tr '-_' '+/' | base64 -d 2>/dev/null | jq\n}\njwt_decode \"$TOKEN\""
      },
      {
        "title": "In code",
        "body": "// JavaScript (no library needed for decoding)\nfunction decodeJWT(token) {\n    const [header, payload] = token.split('.').slice(0, 2)\n        .map(part => JSON.parse(atob(part.replace(/-/g, '+').replace(/_/g, '/'))));\n    return { header, payload };\n}\n\n// Check expiry\nfunction isJWTExpired(token) {\n    const { payload } = decodeJWT(token);\n    return payload.exp && payload.exp < Math.floor(Date.now() / 1000);\n}\n\n# Python\nimport json, base64\n\ndef decode_jwt(token):\n    parts = token.split('.')\n    # Add padding\n    def pad(s): return s + '=' * (4 - len(s) % 4)\n    header = json.loads(base64.urlsafe_b64decode(pad(parts[0])))\n    payload = json.loads(base64.urlsafe_b64decode(pad(parts[1])))\n    return header, payload\n\nheader, payload = decode_jwt(token)"
      },
      {
        "title": "Common hash functions",
        "body": "# MD5 (not for security — only for checksums/dedup)\necho -n \"Hello\" | md5sum        # Linux\necho -n \"Hello\" | md5           # macOS\n\n# SHA-256 (standard for integrity)\necho -n \"Hello\" | sha256sum\necho -n \"Hello\" | shasum -a 256\n\n# SHA-1 (deprecated for security, still used in git)\necho -n \"Hello\" | sha1sum\n\n# SHA-512\necho -n \"Hello\" | sha512sum\n\n# Hash a file\nsha256sum file.bin\nmd5sum file.bin\n\n# openssl (works everywhere)\necho -n \"Hello\" | openssl dgst -sha256\nopenssl dgst -sha256 file.bin"
      },
      {
        "title": "In code",
        "body": "// Node.js\nconst crypto = require('crypto');\ncrypto.createHash('sha256').update('Hello').digest('hex');\n// \"185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969\"\n\n// File hash\nconst fs = require('fs');\nconst hash = crypto.createHash('sha256');\nhash.update(fs.readFileSync('file.bin'));\nconsole.log(hash.digest('hex'));\n\nimport hashlib\nhashlib.sha256(b\"Hello\").hexdigest()\n# \"185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969\"\n\n# File hash\nwith open(\"file.bin\", \"rb\") as f:\n    print(hashlib.sha256(f.read()).hexdigest())"
      },
      {
        "title": "Checksums for file integrity",
        "body": "# Generate checksum file\nsha256sum *.tar.gz > checksums.sha256\n\n# Verify checksums\nsha256sum -c checksums.sha256\n\n# Compare two files without reading content\nsha256sum file1.bin file2.bin\n# or\ncmp file1.bin file2.bin && echo \"Identical\" || echo \"Different\""
      },
      {
        "title": "JSON ↔ other formats",
        "body": "# JSON to YAML\npython3 -c \"import json, yaml, sys; yaml.dump(json.load(sys.stdin), sys.stdout)\" < data.json\n\n# YAML to JSON\npython3 -c \"import json, yaml, sys; json.dump(yaml.safe_load(sys.stdin), sys.stdout, indent=2)\" < data.yaml\n\n# JSON to CSV\njq -r '.[] | [.id, .name, .email] | @csv' data.json > data.csv\n\n# CSV to JSON\npython3 -c \"\nimport csv, json, sys\nreader = csv.DictReader(open(sys.argv[1]))\nprint(json.dumps(list(reader), indent=2))\n\" data.csv\n\n# JSON to TOML\npython3 -c \"import json, tomli_w, sys; tomli_w.dump(json.load(sys.stdin), sys.stdout.buffer)\" < data.json\n\n# Pretty-print JSON\njq '.' data.json\npython3 -m json.tool data.json"
      },
      {
        "title": "Binary formats (inspection)",
        "body": "# MessagePack → JSON\npython3 -c \"\nimport msgpack, json, sys\ndata = msgpack.unpackb(sys.stdin.buffer.read(), raw=False)\nprint(json.dumps(data, indent=2))\n\" < data.msgpack\n\n# Protobuf (decode without schema — shows field numbers)\nprotoc --decode_raw < data.pb\n\n# CBOR → JSON\npython3 -c \"\nimport cbor2, json, sys\ndata = cbor2.loads(sys.stdin.buffer.read())\nprint(json.dumps(data, indent=2, default=str))\n\" < data.cbor"
      },
      {
        "title": "Quick Decode Script",
        "body": "#!/bin/bash\n# decode.sh — Auto-detect and decode common encoded strings\nINPUT=\"${1:-$(cat)}\"\n\n# Try Base64\nB64_DECODED=$(echo \"$INPUT\" | base64 -d 2>/dev/null)\nif [[ $? -eq 0 && -n \"$B64_DECODED\" ]]; then\n    echo \"Base64 → $B64_DECODED\"\nfi\n\n# Try URL encoding\nif echo \"$INPUT\" | grep -q '%[0-9A-Fa-f]\\{2\\}'; then\n    URL_DECODED=$(python3 -c \"from urllib.parse import unquote; print(unquote('$INPUT'))\" 2>/dev/null)\n    echo \"URL   → $URL_DECODED\"\nfi\n\n# Try JWT\nif echo \"$INPUT\" | grep -qP '^eyJ[A-Za-z0-9_-]+\\.eyJ[A-Za-z0-9_-]+\\.'; then\n    echo \"JWT header:\"\n    echo \"$INPUT\" | cut -d. -f1 | tr '-_' '+/' | base64 -d 2>/dev/null | jq\n    echo \"JWT payload:\"\n    echo \"$INPUT\" | cut -d. -f2 | tr '-_' '+/' | base64 -d 2>/dev/null | jq\nfi\n\n# Try hex\nif echo \"$INPUT\" | grep -qP '^[0-9a-fA-F]+$' && [[ $((${#INPUT} % 2)) -eq 0 ]]; then\n    HEX_DECODED=$(echo \"$INPUT\" | xxd -r -p 2>/dev/null)\n    if [[ -n \"$HEX_DECODED\" ]]; then\n        echo \"Hex   → $HEX_DECODED\"\n    fi\nfi"
      },
      {
        "title": "Tips",
        "body": "Base64 increases data size by ~33%. Use it for embedding binary data in text formats (JSON, XML, email), not for compression or encryption.\nBase64url (RFC 4648) uses - and _ instead of + and /, and omits padding =. JWTs and URL parameters use this variant.\nSHA-256 is the standard for integrity checks. MD5 is fine for dedup and non-security checksums but broken for cryptographic use.\nJWTs are signed, not encrypted. Anyone can decode the header and payload. Only the signature verifies authenticity. Never put secrets in JWT claims.\nWhen files display garbled text (mojibake), the problem is almost always wrong encoding assumption. Check with file -bi and re-read with the correct encoding.\nxxd -p (plain hex) and xxd -r -p (reverse) are the fastest way to convert between binary and hex on the command line.\nURL-encode with encodeURIComponent (JavaScript) or urllib.parse.quote (Python), not by hand. Manual encoding misses edge cases."
      }
    ],
    "body": "Encoding & Formats\n\nEncode, decode, and inspect data in common formats. Covers Base64, URL encoding, hex, Unicode, JWTs, hashing, checksums, and serialization formats.\n\nWhen to Use\nDecoding a Base64 string from an API response or config\nURL-encoding parameters for HTTP requests\nInspecting hex dumps of binary data\nDecoding JWT tokens to see claims\nComputing or verifying file checksums\nConverting between character encodings (UTF-8, Latin-1, etc.)\nUnderstanding wire formats (protobuf, MessagePack)\nBase64\nEncode and decode\n# Encode string\necho -n \"Hello, World!\" | base64\n# SGVsbG8sIFdvcmxkIQ==\n\n# Decode string\necho \"SGVsbG8sIFdvcmxkIQ==\" | base64 -d\n# Hello, World!\n\n# Encode a file\nbase64 image.png > image.b64\ncat file.bin | base64\n\n# Decode a file\nbase64 -d image.b64 > image.png\n\n# Base64url (URL-safe variant: + → -, / → _, no padding)\necho -n \"Hello\" | base64 | tr '+/' '-_' | tr -d '='\n# Base64url decode\necho \"SGVsbG8\" | tr '-_' '+/' | base64 -d\n\nIn code\n// JavaScript (browser + Node.js 16+)\nbtoa('Hello');                    // \"SGVsbG8=\"\natob('SGVsbG8=');                 // \"Hello\"\n\n// Node.js Buffer\nBuffer.from('Hello').toString('base64');           // \"SGVsbG8=\"\nBuffer.from('SGVsbG8=', 'base64').toString();      // \"Hello\"\n\n// Binary data\nBuffer.from(binaryData).toString('base64');\nBuffer.from(b64String, 'base64');\n\n# Python\nimport base64\n\nbase64.b64encode(b\"Hello\").decode()     # \"SGVsbG8=\"\nbase64.b64decode(\"SGVsbG8=\")            # b\"Hello\"\n\n# URL-safe Base64\nbase64.urlsafe_b64encode(b\"Hello\").decode()\nbase64.urlsafe_b64decode(\"SGVsbG8=\")\n\nURL Encoding\nEncode and decode\n# Python one-liner\npython3 -c \"from urllib.parse import quote; print(quote('hello world & foo=bar'))\"\n# hello%20world%20%26%20foo%3Dbar\n\n# Decode\npython3 -c \"from urllib.parse import unquote; print(unquote('hello%20world%20%26%20foo%3Dbar'))\"\n# hello world & foo=bar\n\n# curl does it automatically for --data-urlencode\ncurl -G --data-urlencode \"q=hello world & more\" https://api.example.com/search\n\nIn code\n// JavaScript\nencodeURIComponent('hello world & foo=bar');\n// \"hello%20world%20%26%20foo%3Dbar\"\n\ndecodeURIComponent('hello%20world%20%26%20foo%3Dbar');\n// \"hello world & foo=bar\"\n\n// encodeURI vs encodeURIComponent:\nencodeURI('https://example.com/path?q=hello world');\n// \"https://example.com/path?q=hello%20world\" (preserves URL structure)\nencodeURIComponent('https://example.com/path?q=hello world');\n// \"https%3A%2F%2Fexample.com%2Fpath%3Fq%3Dhello%20world\" (encodes everything)\n\nfrom urllib.parse import quote, unquote, urlencode\n\nquote('hello world')            # 'hello%20world'\nunquote('hello%20world')        # 'hello world'\nurlencode({'q': 'hello world', 'page': 1})  # 'q=hello+world&page=1'\n\nHex\nView and convert\n# File hex dump\nxxd file.bin | head -20\nxxd -l 64 file.bin          # First 64 bytes only\n\n# Hex dump (compact, no ASCII)\nxxd -p file.bin\n\n# Convert hex to binary\necho \"48656c6c6f\" | xxd -r -p\n# Hello\n\n# od (alternative)\nod -A x -t x1z file.bin | head -20\n\n# hexdump\nhexdump -C file.bin | head -20\n\n# Python\npython3 -c \"print(bytes.fromhex('48656c6c6f').decode())\"  # Hello\npython3 -c \"print('Hello'.encode().hex())\"                 # 48656c6c6f\n\nIn code\n// JavaScript\nBuffer.from('Hello').toString('hex');           // \"48656c6c6f\"\nBuffer.from('48656c6c6f', 'hex').toString();    // \"Hello\"\n\n// Number to hex\n(255).toString(16);     // \"ff\"\nparseInt('ff', 16);     // 255\n\n# Python\n\"Hello\".encode().hex()                  # '48656c6c6f'\nbytes.fromhex('48656c6c6f').decode()    # 'Hello'\nhex(255)                                # '0xff'\nint('ff', 16)                           # 255\n\nUnicode\nInspect characters\n# Show Unicode code points\necho -n \"Hello 世界\" | python3 -c \"\nimport sys\nfor char in sys.stdin.read():\n    print(f'U+{ord(char):04X}  {char}  {char.encode(\\\"utf-8\\\").hex()}')\"\n# U+0048  H  48\n# U+0065  e  65\n# ...\n# U+4E16  世  e4b896\n# U+754C  界  e7958c\n\n# Convert Unicode escape to character\nprintf '\\u0048\\u0065\\u006c\\u006c\\u006f'   # Hello\necho -e '\\xE4\\xB8\\x96\\xE7\\x95\\x8C'       # 世界\n\n# File encoding detection\nfile -bi document.txt\n# text/plain; charset=utf-8\n\nEncoding conversion\n# Convert between encodings\niconv -f ISO-8859-1 -t UTF-8 input.txt > output.txt\niconv -f UTF-16 -t UTF-8 input.txt > output.txt\n\n# List available encodings\niconv -l\n\n# Python\npython3 -c \"\nwith open('latin1.txt', 'r', encoding='iso-8859-1') as f:\n    content = f.read()\nwith open('utf8.txt', 'w', encoding='utf-8') as f:\n    f.write(content)\n\"\n\nCommon Unicode issues\nBOM (Byte Order Mark):\n  UTF-8 BOM: EF BB BF at start of file\n  Remove: sed -i '1s/^\\xEF\\xBB\\xBF//' file.txt\n\nNormalization (NFC vs NFD):\n  \"é\" can be U+00E9 (one char) or U+0065 U+0301 (e + combining accent)\n  Python: import unicodedata; unicodedata.normalize('NFC', text)\n\nMojibake (wrong encoding):\n  \"café\" appears as \"café\" → file is UTF-8 but read as Latin-1\n  Fix: re-read with correct encoding\n\nJWT (JSON Web Tokens)\nDecode a JWT\n# JWT has 3 parts separated by dots: header.payload.signature\n# Each part is Base64url-encoded\n\n# Decode header and payload\nTOKEN=\"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIiwibmFtZSI6IkpvaG4gRG9lIiwiaWF0IjoxNTE2MjM5MDIyfQ.SflKxwRJSMeKKF2QT4fwpMeJf36POk6yJV_adQssw5c\"\n\n# Decode header\necho \"$TOKEN\" | cut -d. -f1 | tr '-_' '+/' | base64 -d 2>/dev/null | jq\n# {\"alg\":\"HS256\",\"typ\":\"JWT\"}\n\n# Decode payload\necho \"$TOKEN\" | cut -d. -f2 | tr '-_' '+/' | base64 -d 2>/dev/null | jq\n# {\"sub\":\"1234567890\",\"name\":\"John Doe\",\"iat\":1516239022}\n\n# One-liner function\njwt_decode() {\n    echo \"$1\" | cut -d. -f2 | tr '-_' '+/' | base64 -d 2>/dev/null | jq\n}\njwt_decode \"$TOKEN\"\n\nIn code\n// JavaScript (no library needed for decoding)\nfunction decodeJWT(token) {\n    const [header, payload] = token.split('.').slice(0, 2)\n        .map(part => JSON.parse(atob(part.replace(/-/g, '+').replace(/_/g, '/'))));\n    return { header, payload };\n}\n\n// Check expiry\nfunction isJWTExpired(token) {\n    const { payload } = decodeJWT(token);\n    return payload.exp && payload.exp < Math.floor(Date.now() / 1000);\n}\n\n# Python\nimport json, base64\n\ndef decode_jwt(token):\n    parts = token.split('.')\n    # Add padding\n    def pad(s): return s + '=' * (4 - len(s) % 4)\n    header = json.loads(base64.urlsafe_b64decode(pad(parts[0])))\n    payload = json.loads(base64.urlsafe_b64decode(pad(parts[1])))\n    return header, payload\n\nheader, payload = decode_jwt(token)\n\nHashing\nCommon hash functions\n# MD5 (not for security — only for checksums/dedup)\necho -n \"Hello\" | md5sum        # Linux\necho -n \"Hello\" | md5           # macOS\n\n# SHA-256 (standard for integrity)\necho -n \"Hello\" | sha256sum\necho -n \"Hello\" | shasum -a 256\n\n# SHA-1 (deprecated for security, still used in git)\necho -n \"Hello\" | sha1sum\n\n# SHA-512\necho -n \"Hello\" | sha512sum\n\n# Hash a file\nsha256sum file.bin\nmd5sum file.bin\n\n# openssl (works everywhere)\necho -n \"Hello\" | openssl dgst -sha256\nopenssl dgst -sha256 file.bin\n\nIn code\n// Node.js\nconst crypto = require('crypto');\ncrypto.createHash('sha256').update('Hello').digest('hex');\n// \"185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969\"\n\n// File hash\nconst fs = require('fs');\nconst hash = crypto.createHash('sha256');\nhash.update(fs.readFileSync('file.bin'));\nconsole.log(hash.digest('hex'));\n\nimport hashlib\nhashlib.sha256(b\"Hello\").hexdigest()\n# \"185f8db32271fe25f561a6fc938b2e264306ec304eda518007d1764826381969\"\n\n# File hash\nwith open(\"file.bin\", \"rb\") as f:\n    print(hashlib.sha256(f.read()).hexdigest())\n\nChecksums for file integrity\n# Generate checksum file\nsha256sum *.tar.gz > checksums.sha256\n\n# Verify checksums\nsha256sum -c checksums.sha256\n\n# Compare two files without reading content\nsha256sum file1.bin file2.bin\n# or\ncmp file1.bin file2.bin && echo \"Identical\" || echo \"Different\"\n\nSerialization Formats\nJSON ↔ other formats\n# JSON to YAML\npython3 -c \"import json, yaml, sys; yaml.dump(json.load(sys.stdin), sys.stdout)\" < data.json\n\n# YAML to JSON\npython3 -c \"import json, yaml, sys; json.dump(yaml.safe_load(sys.stdin), sys.stdout, indent=2)\" < data.yaml\n\n# JSON to CSV\njq -r '.[] | [.id, .name, .email] | @csv' data.json > data.csv\n\n# CSV to JSON\npython3 -c \"\nimport csv, json, sys\nreader = csv.DictReader(open(sys.argv[1]))\nprint(json.dumps(list(reader), indent=2))\n\" data.csv\n\n# JSON to TOML\npython3 -c \"import json, tomli_w, sys; tomli_w.dump(json.load(sys.stdin), sys.stdout.buffer)\" < data.json\n\n# Pretty-print JSON\njq '.' data.json\npython3 -m json.tool data.json\n\nBinary formats (inspection)\n# MessagePack → JSON\npython3 -c \"\nimport msgpack, json, sys\ndata = msgpack.unpackb(sys.stdin.buffer.read(), raw=False)\nprint(json.dumps(data, indent=2))\n\" < data.msgpack\n\n# Protobuf (decode without schema — shows field numbers)\nprotoc --decode_raw < data.pb\n\n# CBOR → JSON\npython3 -c \"\nimport cbor2, json, sys\ndata = cbor2.loads(sys.stdin.buffer.read())\nprint(json.dumps(data, indent=2, default=str))\n\" < data.cbor\n\nQuick Decode Script\n#!/bin/bash\n# decode.sh — Auto-detect and decode common encoded strings\nINPUT=\"${1:-$(cat)}\"\n\n# Try Base64\nB64_DECODED=$(echo \"$INPUT\" | base64 -d 2>/dev/null)\nif [[ $? -eq 0 && -n \"$B64_DECODED\" ]]; then\n    echo \"Base64 → $B64_DECODED\"\nfi\n\n# Try URL encoding\nif echo \"$INPUT\" | grep -q '%[0-9A-Fa-f]\\{2\\}'; then\n    URL_DECODED=$(python3 -c \"from urllib.parse import unquote; print(unquote('$INPUT'))\" 2>/dev/null)\n    echo \"URL   → $URL_DECODED\"\nfi\n\n# Try JWT\nif echo \"$INPUT\" | grep -qP '^eyJ[A-Za-z0-9_-]+\\.eyJ[A-Za-z0-9_-]+\\.'; then\n    echo \"JWT header:\"\n    echo \"$INPUT\" | cut -d. -f1 | tr '-_' '+/' | base64 -d 2>/dev/null | jq\n    echo \"JWT payload:\"\n    echo \"$INPUT\" | cut -d. -f2 | tr '-_' '+/' | base64 -d 2>/dev/null | jq\nfi\n\n# Try hex\nif echo \"$INPUT\" | grep -qP '^[0-9a-fA-F]+$' && [[ $((${#INPUT} % 2)) -eq 0 ]]; then\n    HEX_DECODED=$(echo \"$INPUT\" | xxd -r -p 2>/dev/null)\n    if [[ -n \"$HEX_DECODED\" ]]; then\n        echo \"Hex   → $HEX_DECODED\"\n    fi\nfi\n\nTips\nBase64 increases data size by ~33%. Use it for embedding binary data in text formats (JSON, XML, email), not for compression or encryption.\nBase64url (RFC 4648) uses - and _ instead of + and /, and omits padding =. JWTs and URL parameters use this variant.\nSHA-256 is the standard for integrity checks. MD5 is fine for dedup and non-security checksums but broken for cryptographic use.\nJWTs are signed, not encrypted. Anyone can decode the header and payload. Only the signature verifies authenticity. Never put secrets in JWT claims.\nWhen files display garbled text (mojibake), the problem is almost always wrong encoding assumption. Check with file -bi and re-read with the correct encoding.\nxxd -p (plain hex) and xxd -r -p (reverse) are the fastest way to convert between binary and hex on the command line.\nURL-encode with encodeURIComponent (JavaScript) or urllib.parse.quote (Python), not by hand. Manual encoding misses edge cases."
  },
  "trust": {
    "sourceLabel": "tencent",
    "provenanceUrl": "https://clawhub.ai/gitgoodordietrying/encoding-formats",
    "publisherUrl": "https://clawhub.ai/gitgoodordietrying/encoding-formats",
    "owner": "gitgoodordietrying",
    "version": "1.0.0",
    "license": null,
    "verificationStatus": "Indexed source record"
  },
  "links": {
    "detailUrl": "https://openagent3.xyz/skills/encoding-formats",
    "downloadUrl": "https://openagent3.xyz/downloads/encoding-formats",
    "agentUrl": "https://openagent3.xyz/skills/encoding-formats/agent",
    "manifestUrl": "https://openagent3.xyz/skills/encoding-formats/agent.json",
    "briefUrl": "https://openagent3.xyz/skills/encoding-formats/agent.md"
  }
}