{"library":{"repo":"olaflaitinen/knap","description":"A byte level BPE tokenizer written from scratch in pure Mojo. Implements the seven encodings OpenAI models use, runs with no Python interpreter in the process, and publishes its evidence: 191.7 million tokens and every token id checked against the reference implementation.","created_at":"2026-09-07T18:15:30Z","first_seen":"2026-10-01T12:59:32Z","found_by":"description / language","stars":4,"forks":0,"license":"EUPL-1.2","size_kb":1483,"commits":75,"mojo_lines":17421,"mojo_files":55,"tooling":"pixi","mojo_version":"1.0.0","version_source":"lock","version_file":"","version_estimate":"1.0","version_estimate_source":"date","topics":["bpe","byte-pair-encoding","differential-testing","fuzzing","gpt","llm","max","modular","mojo","mojo-lang","nlp","openai","simd","text-processing","tiktoken","tokenizer","unicode"],"stats_as_of":"","open_issues":0,"watchers":0,"archived":false,"state":"","renamed_to":""},"migrations":[{"at":"2026-09-07T18:59:34Z","sha":"571c26a69856b69154455a2b5756240022334192","kind":"mojo_version","from":"","to":"1.0.0"},{"at":"2026-09-07T18:59:34Z","sha":"571c26a69856b69154455a2b5756240022334192","kind":"tooling","from":"none","to":"pixi"}],"commits":[{"at":"2026-09-07T18:59:34Z","sha":"571c26a69856b69154455a2b5756240022334192","mojo_lines":154,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-07T19:49:44Z","sha":"fe4530d8c52e38d4161a1a3b0849c7f3133cb777","mojo_lines":2362,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T10:02:07Z","sha":"1180de545ad3a69d34bb295abe4aba2870eb4001","mojo_lines":4748,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T10:56:47Z","sha":"fd9a7a0328b3403941facb347e0df3f49cf4ad80","mojo_lines":6840,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:04:58Z","sha":"d810855b45f80ad4f80be68dd5640358195d4d15","mojo_lines":6886,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:04:58Z","sha":"082d5e99e6c2e817a6c6486274b19ce77b7d869e","mojo_lines":7899,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:04:58Z","sha":"bb28bdc0978accf1c74c45a1f8b53287632892ca","mojo_lines":8093,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:04:58Z","sha":"ded323594b493e7ca82ff5ffd05b5bfea57a448d","mojo_lines":8653,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:04:58Z","sha":"3a818a0ec9410c22579b362f63d2b5bffb48be96","mojo_lines":8946,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:04:58Z","sha":"4c7895859481078695c1dc69aaae9860020f8eef","mojo_lines":9896,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:04:59Z","sha":"856a727bdf1d135d832bbfd9de54f821630b5602","mojo_lines":11148,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:04:59Z","sha":"d02d784ce95320e00f5eaa98bb37dd243ab02b3f","mojo_lines":11148,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:04:59Z","sha":"2291e99beeb9325be71c51c7c384e866775b6833","mojo_lines":11148,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:04:59Z","sha":"15876c8e479e698f3358d368f87cd35d8534aa6d","mojo_lines":11148,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:04:59Z","sha":"bccb0b0111170939c2d44b62880bcda3031e1ff5","mojo_lines":11422,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:04:59Z","sha":"8c1f65e4c44d5de874c4cc4f956c093065c98aa1","mojo_lines":11422,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:04:59Z","sha":"489f05c2cf224c433a1196dddf16a2b483594870","mojo_lines":11422,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:04:59Z","sha":"29941ed2532a78f9fc619b84a525f030eb78fd7b","mojo_lines":11422,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:04:59Z","sha":"854354c06a8f378c0d274277f0757afb430e9663","mojo_lines":11422,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:04:59Z","sha":"0107b432dbd494945d191a98362175220977ed1e","mojo_lines":11422,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:04:59Z","sha":"6278deb357962d91abdf352693d72a86901eca3b","mojo_lines":11422,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:04:59Z","sha":"3f0fa15be413fb403090532efc5502a11b0e7ab9","mojo_lines":11422,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:19:38Z","sha":"8255618c8d8d2fe96925415b0e1b31045826dbc7","mojo_lines":11422,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:19:38Z","sha":"7e00eb659cb72fc8fd4a5093ec6ea423fbed57ce","mojo_lines":11422,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:19:39Z","sha":"5fb7f0e320d5d08dfb15ce35c68b0fd29cdd2c7f","mojo_lines":11422,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:29:37Z","sha":"5dc05024f7458a3ef9a9e166b159ed991709b21a","mojo_lines":11422,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:49:21Z","sha":"b070918786c637a32cfe5f40553951ecc93deda3","mojo_lines":11449,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T17:57:58Z","sha":"413264e6da18393ecd4dd20bedd60cdc40454c15","mojo_lines":11449,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T18:12:15Z","sha":"00d0c794eed5941f7cfc21ade9541978287c006d","mojo_lines":11449,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T18:13:46Z","sha":"4c50132f064579b79a9ce2ac96e690ebbe0ff99a","mojo_lines":11449,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-08T18:14:58Z","sha":"9faff20ba464ec602f7150e774df7ac21d0f2eac","mojo_lines":11449,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T07:31:42Z","sha":"f762cdb4d67136f1930e6d4f73aecbd19d22a99d","mojo_lines":12775,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T07:31:42Z","sha":"cd59ada95f3c686cf6a7febd697a9a04a84182dc","mojo_lines":12775,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T07:31:42Z","sha":"64c1d6d437ed177e8afc8d750fabfe2a852b1dda","mojo_lines":12775,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T07:42:53Z","sha":"7f87cd73fb28b03d34551869ea745b068f3eace9","mojo_lines":12775,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T08:11:48Z","sha":"ff3d7bdd7684194f2776bd95c1cbfe88f7f11ad9","mojo_lines":12775,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T08:11:53Z","sha":"45fc6d3df457dd735810cd868554960aea348689","mojo_lines":12775,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T08:13:45Z","sha":"82075426c456e2fa9774416800c8c72fc46cc7ca","mojo_lines":12775,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T08:31:56Z","sha":"b1d2ee25478e99c313c975f669c0f4c3ec5a893f","mojo_lines":12775,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T09:42:02Z","sha":"6ec02ea19181ed4456a22967dac054322ccbc5f5","mojo_lines":12894,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T09:42:02Z","sha":"d7bcacb32132e5f20dcffd914749638520f635a4","mojo_lines":12894,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T09:42:03Z","sha":"71003bfd65912745dd53285e6293bd9889fa6193","mojo_lines":12894,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T09:42:28Z","sha":"fb45b7c0b06df46a86a850cd4d7d31f16c89688c","mojo_lines":12894,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T09:42:28Z","sha":"0814805655782200b0dd20ada550ec396b6fb7ac","mojo_lines":12894,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T11:49:01Z","sha":"a7a45a73d6a09775da678ef8f6294dae6808fed4","mojo_lines":14532,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T17:06:49Z","sha":"8f17fd6aa9da1d8845369ee706f218434972aab3","mojo_lines":14786,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T17:28:34Z","sha":"fa89bae14e8afa9781fcbafaa1da260c950b453f","mojo_lines":14786,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T17:29:45Z","sha":"eebb252a41a6416bd73e1c300d5b3e0551e3e733","mojo_lines":14786,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T18:33:33Z","sha":"04f50b66bae4c013f767828fbe3d5bc1730d367f","mojo_lines":14851,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T18:34:08Z","sha":"7405a026141066e7ff4a7a921bb6db4306919eb2","mojo_lines":14851,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-09T19:05:07Z","sha":"304ec2c092f0e0558f58ff1d3613124009719301","mojo_lines":14851,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-10T08:47:52Z","sha":"eb910bb9a7fc8597b09370acdc0a83ac230027d5","mojo_lines":15586,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-10T09:19:47Z","sha":"899e0442aa33e8b70d7f10c64a33d5524f03e7f7","mojo_lines":16461,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-10T11:04:25Z","sha":"4734f8c6096e73680dd308531579b3e6d7b1f263","mojo_lines":16461,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-10T11:04:47Z","sha":"cc2617a765d58402883cd932ee0a3028a39ff985","mojo_lines":16461,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-10T11:04:49Z","sha":"0982d712a1a5cd2a082351e082ee44c7b164f44e","mojo_lines":16461,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-10T11:04:51Z","sha":"1bcee0c364f39c6c51b89b3f9a3d4281476cde93","mojo_lines":16461,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-10T11:04:52Z","sha":"92853d574ffd12f030d3271bf01565a64d207e41","mojo_lines":16461,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-10T11:04:54Z","sha":"efdf05c43dbe60f25897489870b3e85f09a824b8","mojo_lines":16461,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-10T11:04:56Z","sha":"8342c79ce3f382b528716ed3dd0f984752630a0f","mojo_lines":16461,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-10T11:53:09Z","sha":"bacb031bafbc400452352f3117e8cc3a537f0410","mojo_lines":16461,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-10T12:02:16Z","sha":"0bcc85d01572016993eb7f71f06aefec06b0ab22","mojo_lines":16461,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-10T12:34:28Z","sha":"54afd98f91b086bc4cae57ed4a8a75e1a6db9024","mojo_lines":16565,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-10T12:41:06Z","sha":"3fbf271be5ef3b63d5f1963527ef7b72458fa4d4","mojo_lines":16565,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-10T15:04:39Z","sha":"3fde614cd2a7241f2dacd5cfd16fa7249fe30429","mojo_lines":17324,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-10T15:05:09Z","sha":"3bc1e184e16f40a8a5d52d3397edeb05274e41a9","mojo_lines":17421,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-10T15:05:25Z","sha":"3927e1dbf3941ff4fabf99e721a6f7cdbc0b76fd","mojo_lines":17421,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-10T15:05:44Z","sha":"3a2ce503094d588b09fa6cb8b4bd180a4b1c4f41","mojo_lines":17421,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-10T17:36:27Z","sha":"0008794d5b1089fb586e84f6aa6779f9b053cf92","mojo_lines":17421,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-10T17:36:59Z","sha":"d90b59d4c4f294c6d09a4590e85fda6bf6a6a481","mojo_lines":17421,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-11T04:26:42Z","sha":"e83f0ea34fb6d586551b7f9df226006d78dc1436","mojo_lines":17421,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-11T04:27:07Z","sha":"05aceb717aa4eb506cb7bc4144761fdb144ee149","mojo_lines":17421,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-11T05:01:35Z","sha":"4c5b07a909c5f56fedc99bd336854d1e3bf6f83b","mojo_lines":17421,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-11T05:08:48Z","sha":"67056e08fd8dc364f5325284daaa3396bc737759","mojo_lines":17421,"tooling":"pixi","mojo_version":"1.0.0"},{"at":"2026-09-11T05:13:24Z","sha":"b1c30f3ab82ba3d328524ffcc3528c6fafb3a1e4","mojo_lines":17421,"tooling":"pixi","mojo_version":"1.0.0"}],"activity":[{"month":"2026-09","stars":2,"forks":0,"releases":0,"issues_opened":0,"prs_opened":0,"pushes":0,"days_active":2},{"month":"2026-10","stars":0,"forks":0,"releases":0,"issues_opened":0,"prs_opened":0,"pushes":0,"days_active":1}],"releases":[{"tag":"v1.0.0","name":"","at":"2026-09-11T05:03:41Z","prerelease":false,"source":"tag","commit":"4c5b07a909c5f56fedc99bd336854d1e3bf6f83b","mojo_version":"1.0.0"}]}