test_amharic.py

#!/usr/bin/env python3
"""Test the EthioBBPE tokenizer with Amharic texts."""

from tokenizers import Tokenizer

# Load the trained tokenizer
tokenizer = Tokenizer.from_file('models/EthioBBPE_AmharicBible/tokenizer.json')

print('='*70)
print('TESTING AMHARIC TEXTS')
print('='*70)

# Test 1: Special Ge'ez punctuation
test1 = '፠፠፠፠፠፠፠፠፠፠፠፠፠፠፠፠፠፠'
encoded1 = tokenizer.encode(test1)
decoded1 = tokenizer.decode(encoded1.ids)
print(f'\nTest 1 - Geez Punctuation:')
print(f'Input:    {test1}')
print(f'Tokens:   {encoded1.tokens}')
print(f'Decoded:  {decoded1}')
print(f'Match:    {test1 == decoded1}')

# Test 2: Biblical text from Synaxarium
test2 = '''ሰላም ለኢዮብ ዘኢነበበ ከንቶ ። አመ አኀዞ አበቅ ወአመ አህጎለ ጥሪቶ ። ሐዋርያ መንፈስ ይቤ እንዘ ያነክር ሕይወቶ ። ናስተብፅዖሙ ናሁ በብዙኅ አዕⷈቶ ። ለዕለ ተዓገሡ ሰብእ ለኢዮብ ትዕግስቶ ።'''
encoded2 = tokenizer.encode(test2)
decoded2 = tokenizer.decode(encoded2.ids)
print(f'\nTest 2 - Biblical Text (Synaxarium):')
print(f'Input:    {test2[:80]}...')
print(f'Tokens:   {encoded2.tokens[:20]}... ({len(encoded2.tokens)} total)')
print(f'Decoded:  {decoded2[:80]}...')
print(f'Match:    {test2 == decoded2}')

# Test 3: Another biblical passage
test3 = 'ወደ ቍስጥንጥንያ አገርም በደረሰች ጊዜ ያቺ ሴት ወደ ንጉሡ ሒዳ የቅዱስ እስጢፋኖስን ዜና ከእርሱ የሆኑትን ተአምራት ወደ ቍስጥንጥንያ አገር ወደብም እንደ ደረሰ ነገረችው ሰምቶም እጅግ ደስ አለው'
encoded3 = tokenizer.encode(test3)
decoded3 = tokenizer.decode(encoded3.ids)
print(f'\nTest 3 - Canon Biblical Text:')
print(f'Input:    {test3[:60]}...')
print(f'Tokens:   {encoded3.tokens[:15]}... ({len(encoded3.tokens)} total)')
print(f'Decoded:  {decoded3[:60]}...')
print(f'Match:    {test3 == decoded3}')

# Overall assessment
all_match = (test1 == decoded1) and (test2 == decoded2) and (test3 == decoded3)
print('\n' + '='*70)
if all_match:
    print('RESULT: PERFECT - All texts reconstructed exactly!')
    print('The tokenizer is ready for production use.')
else:
    print('RESULT: NEEDS IMPROVEMENT - Some texts not perfectly reconstructed.')
    print('Consider retraining with more data or larger vocabulary.')
print('='*70)