import json
from tokenizer import encode

# Load full resume text
with open("resume_text.txt", "r", encoding="utf-8") as f:
    text = f.read()

data = encode(text)
context_length = 64

def get_pairs(data, context_length):
    pairs = []
    for i in range(len(data) - context_length):
        x = data[i : i + context_length]
        y = data[i + 1 : i + context_length + 1]
        pairs.append((x, y))
    return pairs

pairs = get_pairs(data, context_length)

print(f"Total characters encoded : {len(data)}")
print(f"Total training pairs     : {len(pairs)}")
print(f"Sample input  : {pairs[0][0][:10]} ...")
print(f"Sample target : {pairs[0][1][:10]} ...")