{ "_id": "6abd040099f7aa8c6aa4eba5", "Package": "rtiktoken", "Title": "A Byte-Pair-Encoding (BPE) Tokenizer for OpenAI's Large Language\nModels", "Version": "0.11.0.3", "Authors@R": "c(\nperson(\"David\", \"Zimmermann-Kollenda\", , \"david_j_zimmermann@hotmail.com\", role = c(\"aut\", \"cre\")),\nperson(\"Roger Zurawicki\", role = \"aut\", comment = \"tiktoken-rs Rust library\"),\nperson(\"Authors of the dependent Rust crates\", role = \"aut\", comment = \"see AUTHORS file\")\n)", "Description": "A thin wrapper around the tiktoken-rs crate, allowing to\nencode text into Byte-Pair-Encoding (BPE) tokens and decode\ntokens back to text. This is useful to understand how Large\nLanguage Models (LLMs) perceive text.", "License": "MIT + file LICENSE", "URL": "https://davzim.github.io/rtiktoken/,\nhttps://github.com/DavZim/rtiktoken/", "BugReports": "https://github.com/DavZim/rtiktoken/issues", "SystemRequirements": "Cargo (Rust's package manager), rustc >= 1.65.0", "Encoding": "UTF-8", "Roxygen": "list(markdown = TRUE)", "RoxygenNote": "7.3.3", "Config/rextendr/version": "0.5.0", "Config/testthat/edition": "3", "Config/rtiktoken/MSRV": "1.65.0", "Config/pak/sysreqs": "libclang-dev", "Repository": "https://davzim.r-universe.dev", "Date/Publication": "2026-08-31 12:07:54 UTC", "RemoteUrl": "https://github.com/davzim/rtiktoken", "RemoteRef": "HEAD", "RemoteSha": "73ffc2d6f3a7f3a9f0855bdf01555c853c887247", "NeedsCompilation": "yes", "Packaged": { "Date": "2026-09-30 12:35:47 UTC", "User": "root" }, "Author": "David Zimmermann-Kollenda [aut, cre],\nRoger Zurawicki [aut] (tiktoken-rs Rust library),\nAuthors of the dependent Rust crates [aut] (see AUTHORS file)", "Maintainer": "David Zimmermann-Kollenda ", "_user": "davzim", "_type": "src", "_file": "rtiktoken_0.11.0.3.tar.gz", "_fileid": "https://r2.ropensci.org/711e00e6624285f5b9c623f5b77e8936dd3f97de07c5aa170a62769a426be3a4", "_filesize": 4275142, "_sha256": "711e00e6624285f5b9c623f5b77e8936dd3f97de07c5aa170a62769a426be3a4", "_expires": "2027-01-08T12:43:41.000Z", "_created": "2026-09-30T12:35:47.000Z", "_published": "2026-09-30T12:43:44.220Z", "_jobs": [ { "job": 109888423357, "time": 164, "config": "linux-devel-arm64", "r": "4.7.0", "check": "OK", "artifact": "11096870664" }, { "job": 109888423301, "time": 141, "config": "linux-devel-x86_64", "r": "4.7.0", "check": "OK", "artifact": "11095822457" }, { "job": 109888423350, "time": 142, "config": "linux-release-arm64", "r": "4.6.1", "check": "OK", "artifact": "11096496068" }, { "job": 109888423323, "time": 156, "config": "linux-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "11096995572" }, { "job": 109888423364, "time": 205, "config": "macos-oldrel-arm64", "r": "4.5.3", "check": "OK", "artifact": "11096326384" }, { "job": 109888423510, "time": 341, "config": "macos-oldrel-x86_64", "r": "4.5.3", "check": "OK", "artifact": "11096167147" }, { "job": 109888423398, "time": 269, "config": "macos-release-arm64", "r": "4.6.1", "check": "OK", "artifact": "11096686072" }, { "job": 109888423437, "time": 335, "config": "macos-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "11096346515" }, { "job": 109886858562, "time": 251, "config": "source", "r": "4.6.1", "check": "OK", "artifact": "11096265944" }, { "job": 109888423169, "time": 142, "config": "wasm-release", "r": "4.6.0", "check": "OK", "artifact": "11096266353" }, { "job": 109888423499, "time": 399, "config": "windows-devel-arm64", "r": "4.7.0", "check": "OK", "artifact": "11096606579" }, { "job": 109888423814, "time": 190, "config": "windows-devel-x86_64", "r": "4.7.0", "check": "OK", "artifact": "11096730646" }, { "job": 109888423521, "time": 150, "config": "windows-oldrel-x86_64", "r": "4.5.3", "check": "OK", "artifact": "11096730538" }, { "job": 109888423352, "time": 408, "config": "windows-release-arm64", "r": "4.6.1", "check": "OK", "artifact": "11096012812" }, { "job": 109888423603, "time": 163, "config": "windows-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "11095434875" } ], "_host": "GitHub-Actions", "_buildurl": "https://github.com/r-universe/davzim/actions/runs/36715333360", "_status": "success", "_upstream": "https://github.com/davzim/rtiktoken", "_commit": { "id": "73ffc2d6f3a7f3a9f0855bdf01555c853c887247", "author": "DavZim ", "committer": "DavZim ", "message": "submit to CRAN\n", "time": 1788178074 }, "_maintainer": { "name": "David Zimmermann-Kollenda", "email": "david_j_zimmermann@hotmail.com" }, "_distro": "resolute", "_registered": true, "_dependencies": [ { "package": "R", "version": ">= 4.2", "role": "Depends" }, { "package": "testthat", "version": ">= 3.0.0", "role": "Suggests" } ], "_owner": "davzim", "_selfowned": true, "_usedby": 0, "_updates": [ { "week": "2026-15", "n": 2 }, { "week": "2026-17", "n": 2 }, { "week": "2026-34", "n": 4 }, { "week": "2026-36", "n": 2 } ], "_tags": [], "_topics": [ "bpe", "openai", "rust", "tokenization", "cargo" ], "_stars": 14, "_contributors": [ { "user": "davzim", "count": 33, "uuid": 15910496 }, { "user": "josiahparry", "count": 1, "uuid": 13534508 } ], "_userbio": { "uuid": 15910496, "type": "user", "name": "DavZim", "followers": 71 }, "_downloads": { "count": 331, "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/rtiktoken" }, "_devurl": "https://github.com/davzim/rtiktoken", "_pkgdown": "https://davzim.github.io/rtiktoken/", "_searchresults": 7, "_cargo": true, "_rbuild": "4.6.1", "_assets": [ "extra/citation.cff", "extra/citation.html", "extra/citation.json", "extra/citation.txt", "extra/contents.json", "extra/manual.pdf", "extra/NEWS.html", "extra/NEWS.txt", "extra/readme.html", "extra/readme.md", "extra/rtiktoken.html", "LICENSE" ], "_homeurl": "https://github.com/davzim/rtiktoken", "_realowner": "davzim", "_cranurl": true, "_releases": [ { "version": "0.0.6", "date": "2024-11-06" }, { "version": "0.0.7", "date": "2025-04-14" }, { "version": "0.11.0.2", "date": "2026-08-21" }, { "version": "0.11.0.3", "date": "2026-09-08" } ], "_exports": [ "decode_tokens", "get_token_count", "get_tokens", "model_to_tokenizer" ], "_help": [ { "page": "decode_tokens", "title": "Decodes tokens back to text", "topics": [ "decode_tokens" ] }, { "page": "get_token_count", "title": "Returns the number of tokens in a text", "topics": [ "get_token_count" ] }, { "page": "get_tokens", "title": "Converts text to tokens", "topics": [ "get_tokens" ] }, { "page": "model_to_tokenizer", "title": "Gets the name of the tokenizer used by a model", "topics": [ "model_to_tokenizer" ] } ], "_readme": "https://github.com/davzim/rtiktoken/raw/HEAD/README.md", "_rundeps": [], "_score": 4.447158031342219, "_indexed": true, "_nocasepkg": "rtiktoken", "_universes": [ "davzim" ], "_binaries": [ { "r": "4.7.0", "os": "linux", "version": "0.11.0.3", "date": "2026-09-30T12:38:42.000Z", "distro": "resolute", "arch": "aarch64", "commit": "73ffc2d6f3a7f3a9f0855bdf01555c853c887247", "fileid": "https://r2.ropensci.org/36ea6065f7c96366147b5ac6312a42a548f14e8805754b5ef34abe31b25cceb6", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/36715333360" }, { "r": "4.7.0", "os": "linux", "version": "0.11.0.3", "date": "2026-09-30T12:38:18.000Z", "distro": "resolute", "arch": "x86_64", "commit": "73ffc2d6f3a7f3a9f0855bdf01555c853c887247", "fileid": "https://r2.ropensci.org/85ca7c9afb73997ae33d6fd0e0bfd6431c9957de5473732f09a651c105c3a7dc", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/36715333360" }, { "r": "4.6.1", "os": "linux", "version": "0.11.0.3", "date": "2026-09-30T12:38:24.000Z", "distro": "resolute", "arch": "aarch64", "commit": "73ffc2d6f3a7f3a9f0855bdf01555c853c887247", "fileid": "https://r2.ropensci.org/4c28ebf168886e678574665ce711bcb36e22f14cf5f221553aed433e16f860be", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/36715333360" }, { "r": "4.6.1", "os": "linux", "version": "0.11.0.3", "date": "2026-09-30T12:38:28.000Z", "distro": "resolute", "arch": "x86_64", "commit": "73ffc2d6f3a7f3a9f0855bdf01555c853c887247", "fileid": "https://r2.ropensci.org/60f3a391ea9c926b588ce6be5afb51f752850816470b6c0ff3e67a42bc464279", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/36715333360" }, { "r": "4.5.3", "os": "mac", "version": "0.11.0.3", "date": "2026-09-30T12:38:46.000Z", "arch": "aarch64", "commit": "73ffc2d6f3a7f3a9f0855bdf01555c853c887247", "fileid": "https://r2.ropensci.org/8c93b016ff105fcdc88cee580120314878240e33a249f30707248d5ff8e24032", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/36715333360" }, { "r": "4.5.3", "os": "mac", "version": "0.11.0.3", "date": "2026-09-30T12:40:15.000Z", "arch": "x86_64", "commit": "73ffc2d6f3a7f3a9f0855bdf01555c853c887247", "fileid": "https://r2.ropensci.org/5556077a52592f96aff119ff6a7303dca367e53ac7cb0922857caa82129e6b76", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/36715333360" }, { "r": "4.6.1", "os": "mac", "version": "0.11.0.3", "date": "2026-09-30T12:39:21.000Z", "arch": "aarch64", "commit": "73ffc2d6f3a7f3a9f0855bdf01555c853c887247", "fileid": "https://r2.ropensci.org/f44f183cf28615e63a9b865496010211ac8e14007871aa05728ed861447fce91", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/36715333360" }, { "r": "4.6.1", "os": "mac", "version": "0.11.0.3", "date": "2026-09-30T12:39:52.000Z", "arch": "x86_64", "commit": "73ffc2d6f3a7f3a9f0855bdf01555c853c887247", "fileid": "https://r2.ropensci.org/0c84c57542cc9606d52000ea1fde361f9cd126c8f8c3af1eed6173e32cefb115", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/36715333360" }, { "r": "4.6.0", "os": "wasm", "version": "0.11.0.3", "date": "2026-09-30T12:38:27.000Z", "arch": "emscripten", "commit": "73ffc2d6f3a7f3a9f0855bdf01555c853c887247", "fileid": "https://r2.ropensci.org/86e1c7b6e62072d66469f713bf0597df464174b7dda94e04f350257fd8cb494a", "status": "success", "buildurl": "https://github.com/r-universe/davzim/actions/runs/36715333360" }, { "r": "4.7.0", "os": "win", "version": "0.11.0.3", "date": "2026-09-30T12:41:22.000Z", "arch": "aarch64", "commit": "73ffc2d6f3a7f3a9f0855bdf01555c853c887247", "fileid": "https://r2.ropensci.org/bc4bd934994a33aaf0b85898444037a00169d085f8b5152bf2409ea3f36f8dac", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/36715333360" }, { "r": "4.7.0", "os": "win", "version": "0.11.0.3", "date": "2026-09-30T12:37:30.000Z", "arch": "x86_64", "commit": "73ffc2d6f3a7f3a9f0855bdf01555c853c887247", "fileid": "https://r2.ropensci.org/c8547b2067e3c47acb9467af8f997210a5a875e3883a2c29cc7b666bf2f1f9f1", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/36715333360" }, { "r": "4.5.3", "os": "win", "version": "0.11.0.3", "date": "2026-09-30T12:37:23.000Z", "arch": "x86_64", "commit": "73ffc2d6f3a7f3a9f0855bdf01555c853c887247", "fileid": "https://r2.ropensci.org/1d3129cb8143203ca33b6dc02d66ae76217cc10abef2e84b1b865ea14cf0cf1e", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/36715333360" }, { "r": "4.6.1", "os": "win", "version": "0.11.0.3", "date": "2026-09-30T12:41:25.000Z", "arch": "aarch64", "commit": "73ffc2d6f3a7f3a9f0855bdf01555c853c887247", "fileid": "https://r2.ropensci.org/07b402032cb6ced35614ecf69e293fd45f4641681c58a79359ae22b4e14fc173", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/36715333360" }, { "r": "4.6.1", "os": "win", "version": "0.11.0.3", "date": "2026-09-30T12:37:27.000Z", "arch": "x86_64", "commit": "73ffc2d6f3a7f3a9f0855bdf01555c853c887247", "fileid": "https://r2.ropensci.org/ca1f6592a27e4e4536b12d62e0c19fd540d22d9acc7d96a652cd8ea49e5b539a", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/36715333360" } ] }