How to Extract Your AI Coding Assistant Chats

# Clone the repo
git clone https://github.com/0xSero/ai-data-extraction
cd ai-data-extraction

# Extract from one tool
python3 extract_claude_code.py

# Extract from all tools
./extract_all.sh

{
  "messages": [
    {
      "role": "user",
      "content": "How do I fix this error?",
      "code_context": [
        {
          "file": "/project/src/app.ts",
          "code": "const x = undefined;",
          "range": {"selectionStartLineNumber": 10}
        }
      ]
    },
    {
      "role": "assistant",
      "content": "The error is because...",
      "suggested_diffs": [...]
    }
  ],
  "source": "cursor-composer",
  "created_at": 1705414222000
}

import json

with open('extracted_data/cursor_complete.jsonl') as f:
    for line in f:
        conv = json.loads(line)
        if any('suggested_diffs' in m for m in conv['messages']):
            print(json.dumps(conv))

from datetime import datetime

cutoff = datetime(2024, 1, 1).timestamp() * 1000
with open('extracted_data/claude_code.jsonl') as f:
    for line in f:
        conv = json.loads(line)
        if conv.get('created_at', 0) > cutoff:
            print(conv['messages'][0]['content'][:100])

cat extracted_data/*.jsonl > all_conversations.jsonl
wc -l all_conversations.jsonl

from unsloth import FastLanguageModel

model, tokenizer = FastLanguageModel.from_pretrained(
    "unsloth/qwen2.5-coder-7b-instruct",
    max_seq_length=4096,
    load_in_4bit=True,
)

def format_chat(example):
    return {
        'text': tokenizer.apply_chat_template(
            example['messages'],
            tokenize=False
        )
    }

# Scan for secrets
pip install detect-secrets
detect-secrets scan extracted_data/*.jsonl