Upload run_training.sh with huggingface_hub

run_training.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+# Quick start script for training EthioBBPE tokenizer on Amharic Bible datasets
+
+set -e
+
+echo "=============================================="
+echo "EthioBBPE Tokenizer Training - Quick Start"
+echo "=============================================="
+
+# Configuration
+DATA_DIR="./data"
+MODEL_NAME="EthioBBPE_AmharicBible"
+VOCAB_SIZE=8000
+COMPRESSION_FORMAT="gzip"
+COMPRESSION_LEVEL=9
+QUANTIZATION_BITS=8
+MAX_CHECKPOINTS=3
+
+echo ""
+echo "Configuration:"
+echo "  Data Directory: $DATA_DIR"
+echo "  Model Name: $MODEL_NAME"
+echo "  Vocabulary Size: $VOCAB_SIZE"
+echo "  Compression: $COMPRESSION_FORMAT (level $COMPRESSION_LEVEL)"
+echo "  Quantization: ${QUANTIZATION_BITS}-bit"
+echo "  Max Checkpoints: $MAX_CHECKPOINTS"
+echo ""
+
+# Step 1: Prepare datasets (if not already done)
+if [ ! -f "$DATA_DIR/combined_corpus.txt" ]; then
+    echo "Step 1: Preparing datasets..."
+    python scripts/prepare_datasets.py --output_dir $DATA_DIR
+else
+    echo "Step 1: Dataset already prepared (combined_corpus.txt exists)"
+fi
+
+echo ""
+echo "Step 2: Training tokenizer..."
+python scripts/train_tokenizer.py \
+    --data_dir $DATA_DIR \
+    --vocab_size $VOCAB_SIZE \
+    --model_name $MODEL_NAME \
+    --compression_format $COMPRESSION_FORMAT \
+    --compression_level $COMPRESSION_LEVEL \
+    --enable_quantization \
+    --quantization_bits $QUANTIZATION_BITS \
+    --max_checkpoints $MAX_CHECKPOINTS \
+    --export_formats tokenizer.json vocab_compressed hf_export quantized \
+    --no_checkpoint
+
+echo ""
+echo "=============================================="
+echo "Training Complete!"
+echo "=============================================="
+echo ""
+echo "Model saved to: models/$MODEL_NAME/"
+echo ""
+echo "Files generated:"
+ls -lh models/$MODEL_NAME/
+echo ""
+echo "Usage example:"
+echo "  from tokenizers import Tokenizer"
+echo "  tokenizer = Tokenizer.from_file('models/$MODEL_NAME/tokenizer.json')"
+echo ""