run_training.sh

#!/bin/bash
# Quick start script for training EthioBBPE tokenizer on Amharic Bible datasets

set -e

echo "=============================================="
echo "EthioBBPE Tokenizer Training - Quick Start"
echo "=============================================="

# Configuration
DATA_DIR="./data"
MODEL_NAME="EthioBBPE_AmharicBible"
VOCAB_SIZE=8000
COMPRESSION_FORMAT="gzip"
COMPRESSION_LEVEL=9
QUANTIZATION_BITS=8
MAX_CHECKPOINTS=3

echo ""
echo "Configuration:"
echo "  Data Directory: $DATA_DIR"
echo "  Model Name: $MODEL_NAME"
echo "  Vocabulary Size: $VOCAB_SIZE"
echo "  Compression: $COMPRESSION_FORMAT (level $COMPRESSION_LEVEL)"
echo "  Quantization: ${QUANTIZATION_BITS}-bit"
echo "  Max Checkpoints: $MAX_CHECKPOINTS"
echo ""

# Step 1: Prepare datasets (if not already done)
if [ ! -f "$DATA_DIR/combined_corpus.txt" ]; then
    echo "Step 1: Preparing datasets..."
    python scripts/prepare_datasets.py --output_dir $DATA_DIR
else
    echo "Step 1: Dataset already prepared (combined_corpus.txt exists)"
fi

echo ""
echo "Step 2: Training tokenizer..."
python scripts/train_tokenizer.py \
    --data_dir $DATA_DIR \
    --vocab_size $VOCAB_SIZE \
    --model_name $MODEL_NAME \
    --compression_format $COMPRESSION_FORMAT \
    --compression_level $COMPRESSION_LEVEL \
    --enable_quantization \
    --quantization_bits $QUANTIZATION_BITS \
    --max_checkpoints $MAX_CHECKPOINTS \
    --export_formats tokenizer.json vocab_compressed hf_export quantized \
    --no_checkpoint

echo ""
echo "=============================================="
echo "Training Complete!"
echo "=============================================="
echo ""
echo "Model saved to: models/$MODEL_NAME/"
echo ""
echo "Files generated:"
ls -lh models/$MODEL_NAME/
echo ""
echo "Usage example:"
echo "  from tokenizers import Tokenizer"
echo "  tokenizer = Tokenizer.from_file('models/$MODEL_NAME/tokenizer.json')"
echo ""