Upload scripts/train_tokenizer.py with huggingface_hub

scripts/train_tokenizer.py

@@ -141,18 +141,11 @@ def parse_args():
         help="Directory to save checkpoints",
     )
     
-    parser.add_argument(
-        "--max_checkpoints",
-        type=int,
-        default=5,
-        help="Maximum number of checkpoints to keep (0 = unlimited)",
-    )
-    
     parser.add_argument(
         "--save_compressed",
         action="store_true",
         default=True,
-        help="Save tokenizer files in compressed format (.gz, .bz2, .xz)",
+        help="Save tokenizer files in compressed format (.gz)",
     )
     
     parser.add_argument(
@@ -162,45 +155,6 @@ def parse_args():
         help="Disable compression",
     )
     
-    parser.add_argument(
-        "--compression_format",
-        type=str,
-        choices=['gzip', 'bz2', 'lzma'],
-        default='gzip',
-        help="Compression format to use",
-    )
-    
-    parser.add_argument(
-        "--compression_level",
-        type=int,
-        choices=range(1, 10),
-        default=9,
-        help="Compression level (1-9, higher = better compression but slower)",
-    )
-    
-    parser.add_argument(
-        "--enable_quantization",
-        action="store_true",
-        default=False,
-        help="Enable model quantization for deployment",
-    )
-    
-    parser.add_argument(
-        "--quantization_bits",
-        type=int,
-        choices=[4, 8],
-        default=8,
-        help="Quantization bits (4 or 8)",
-    )
-    
-    parser.add_argument(
-        "--export_formats",
-        type=str,
-        nargs='+',
-        default=['tokenizer.json', 'vocab_compressed', 'hf_export'],
-        help="Export formats to generate",
-    )
-    
     return parser.parse_args()
 
 
@@ -226,11 +180,6 @@ def main():
             use_checkpoint=args.use_checkpoint,
             checkpoint_dir=args.checkpoint_dir,
             save_compressed=args.save_compressed,
-            compression_format=args.compression_format,
-            compression_level=args.compression_level,
-            enable_quantization=args.enable_quantization,
-            quantization_bits=args.quantization_bits,
-            max_checkpoints=args.max_checkpoints,
         )
     
     # Save config if requested
@@ -260,21 +209,7 @@ def main():
         sys.exit(1)
     
     # Save the tokenizer
-    save_path = trainer.save(export_formats=args.export_formats)
-    
-    # List checkpoints
-    print("\n" + "="*60)
-    print("CHECKPOINTS")
-    print("="*60)
-    checkpoints = trainer.list_checkpoints()
-    if checkpoints:
-        for ckpt in checkpoints:
-            status = "✓" if trainer.validate_checkpoint(ckpt['path']) else "✗"
-            print(f"{status} {ckpt['name']} ({ckpt['size_kb']:.1f} KB)")
-            if 'vocab_size' in ckpt:
-                print(f"   Vocab size: {ckpt['vocab_size']}, Final: {ckpt.get('is_final', False)}")
-    else:
-        print("No checkpoints found.")
+    save_path = trainer.save()
     
     # Test the tokenizer
     print("\n" + "="*60)
@@ -302,24 +237,9 @@ def main():
     print(f"Tokenizer training complete!")
     print(f"Model saved to: {save_path}")
     if args.save_compressed:
-        print(f"Compressed files saved (format: {args.compression_format}, level: {args.compression_level})")
-    if args.enable_quantization:
-        print(f"Quantized model saved ({args.quantization_bits}-bit)")
+        print(f"Compressed files also saved (look for .gz files)")
     if args.use_checkpoint:
-        print(f"Checkpoints saved to: {args.checkpoint_dir} (max: {args.max_checkpoints})")
-    
-    # Print training metrics
-    print("\n" + "="*60)
-    print("TRAINING METRICS")
-    print("="*60)
-    metrics = trainer.get_training_metrics()
-    if 'initial' in metrics:
-        print(f"Files processed: {metrics['initial'].get('num_files', 'N/A')}")
-        print(f"Total data size: {metrics['initial'].get('total_bytes', 0) / 1024:.2f} KB")
-    if 'final' in metrics:
-        print(f"Final vocab size: {metrics['final'].get('vocab_size', 'N/A')}")
-        print(f"Training duration: {metrics['final'].get('training_duration_sec', 0):.2f} seconds")
-    
+        print(f"Checkpoints saved to: {args.checkpoint_dir}")
     print("="*60)