use gpt-oss tokenizer because it's a great tokenizer

This commit is contained in:
Kye Gomez 2026-04-19 22:28:09 -04:00
parent 5cfef742b5
commit 97bc414977
4 changed files with 373 additions and 7 deletions

View file

@ -15,6 +15,7 @@ from open_mythos.main import (
apply_rope,
loop_index_embedding,
)
from open_mythos.tokenizer import MythosTokenizer
from open_mythos.variants import (
mythos_1b,
mythos_3b,
@ -48,4 +49,7 @@ __all__ = [
"mythos_100b",
"mythos_500b",
"mythos_1t",
"load_tokenizer",
"get_vocab_size",
"MythosTokenizer",
]