File size: 1,902 Bytes
b43566f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | #!/bin/bash
# Quick start script for training EthioBBPE tokenizer on Amharic Bible datasets
set -e
echo "=============================================="
echo "EthioBBPE Tokenizer Training - Quick Start"
echo "=============================================="
# Configuration
DATA_DIR="./data"
MODEL_NAME="EthioBBPE_AmharicBible"
VOCAB_SIZE=8000
COMPRESSION_FORMAT="gzip"
COMPRESSION_LEVEL=9
QUANTIZATION_BITS=8
MAX_CHECKPOINTS=3
echo ""
echo "Configuration:"
echo " Data Directory: $DATA_DIR"
echo " Model Name: $MODEL_NAME"
echo " Vocabulary Size: $VOCAB_SIZE"
echo " Compression: $COMPRESSION_FORMAT (level $COMPRESSION_LEVEL)"
echo " Quantization: ${QUANTIZATION_BITS}-bit"
echo " Max Checkpoints: $MAX_CHECKPOINTS"
echo ""
# Step 1: Prepare datasets (if not already done)
if [ ! -f "$DATA_DIR/combined_corpus.txt" ]; then
echo "Step 1: Preparing datasets..."
python scripts/prepare_datasets.py --output_dir $DATA_DIR
else
echo "Step 1: Dataset already prepared (combined_corpus.txt exists)"
fi
echo ""
echo "Step 2: Training tokenizer..."
python scripts/train_tokenizer.py \
--data_dir $DATA_DIR \
--vocab_size $VOCAB_SIZE \
--model_name $MODEL_NAME \
--compression_format $COMPRESSION_FORMAT \
--compression_level $COMPRESSION_LEVEL \
--enable_quantization \
--quantization_bits $QUANTIZATION_BITS \
--max_checkpoints $MAX_CHECKPOINTS \
--export_formats tokenizer.json vocab_compressed hf_export quantized \
--no_checkpoint
echo ""
echo "=============================================="
echo "Training Complete!"
echo "=============================================="
echo ""
echo "Model saved to: models/$MODEL_NAME/"
echo ""
echo "Files generated:"
ls -lh models/$MODEL_NAME/
echo ""
echo "Usage example:"
echo " from tokenizers import Tokenizer"
echo " tokenizer = Tokenizer.from_file('models/$MODEL_NAME/tokenizer.json')"
echo ""
|