First good level of progress

2025-06-29 12:36:25 -04:00
commit 159be1eb82
15 changed files with 10628 additions and 0 deletions
--- a/config.py
+++ b/config.py
@ -0,0 +1,11 @@
+# config.py
+
+VOCAB_SIZE = 50000
+CONTEXT_SIZE = 128
+EMBED_DIM = 256
+NUM_HEADS = 8
+NUM_LAYERS = 6
+BATCH_SIZE = 16
+LEARNING_RATE = 3e-4
+DEVICE = "cuda"  # fallback handled in trainer
+MAX_TOKENS = 100_000  # Used to cap input corpus size