{ "_id": "6a48a18be06c18f0a6543c43", "Package": "rtiktoken", "Title": "A Byte-Pair-Encoding (BPE) Tokenizer for OpenAI's Large Language\nModels", "Version": "0.11.0-1", "Authors@R": "c(\nperson(\"David\", \"Zimmermann-Kollenda\", , \"david_j_zimmermann@hotmail.com\", role = c(\"aut\", \"cre\")),\nperson(\"Roger Zurawicki\", role = \"aut\", comment = \"tiktoken-rs Rust library\"),\nperson(\"Authors of the dependent Rust crates\", role = \"aut\", comment = \"see AUTHORS file\")\n)", "Description": "A thin wrapper around the tiktoken-rs crate, allowing to\nencode text into Byte-Pair-Encoding (BPE) tokens and decode\ntokens back to text. This is useful to understand how Large\nLanguage Models (LLMs) perceive text.", "License": "MIT + file LICENSE", "URL": "https://davzim.github.io/rtiktoken/,\nhttps://github.com/DavZim/rtiktoken/", "BugReports": "https://github.com/DavZim/rtiktoken/issues", "SystemRequirements": "Cargo (Rust's package manager), rustc >= 1.65.0", "Encoding": "UTF-8", "Roxygen": "list(markdown = TRUE)", "RoxygenNote": "7.3.3", "Config/rextendr/version": "0.5.0", "Config/testthat/edition": "3", "Config/rtiktoken/MSRV": "1.65.0", "Config/pak/sysreqs": "libclang-dev", "Repository": "https://davzim.r-universe.dev", "Date/Publication": "2026-04-23 14:06:54 UTC", "RemoteUrl": "https://github.com/davzim/rtiktoken", "RemoteRef": "HEAD", "RemoteSha": "52e7f16a924bdfa255f02670b73452b8af422dcd", "NeedsCompilation": "yes", "Packaged": { "Date": "2026-07-04 05:39:16 UTC", "User": "root" }, "Author": "David Zimmermann-Kollenda [aut, cre],\nRoger Zurawicki [aut] (tiktoken-rs Rust library),\nAuthors of the dependent Rust crates [aut] (see AUTHORS file)", "Maintainer": "David Zimmermann-Kollenda ", "_user": "davzim", "_type": "src", "_file": "rtiktoken_0.11.0-1.tar.gz", "_fileid": "https://r2.ropensci.org/4005d0bcb2e90a3b1c061a56f0b66f701139ca76b2f0109f26695953d3457522", "_filesize": 4274532, "_sha256": "4005d0bcb2e90a3b1c061a56f0b66f701139ca76b2f0109f26695953d3457522", "_expires": "2026-10-12T06:00:41.000Z", "_created": "2026-07-04T05:39:16.000Z", "_published": "2026-07-04T06:00:43.334Z", "_jobs": [ { "job": 85107013173, "time": 159, "config": "linux-devel-arm64", "r": "4.7.0", "check": "OK", "artifact": "8078775233" }, { "job": 85107013178, "time": 163, "config": "linux-devel-x86_64", "r": "4.7.0", "check": "OK", "artifact": "8078775403" }, { "job": 85107013192, "time": 204, "config": "linux-release-arm64", "r": "4.6.1", "check": "OK", "artifact": "8078780803" }, { "job": 85107013171, "time": 162, "config": "linux-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "8078775247" }, { "job": 85107013176, "time": 126, "config": "macos-oldrel-arm64", "r": "4.5.3", "check": "OK", "artifact": "8078902302" }, { "job": 85107013187, "time": 244, "config": "macos-oldrel-x86_64", "r": "4.5.3", "check": "OK", "artifact": "8078794318" }, { "job": 85107013183, "time": 142, "config": "macos-release-arm64", "r": "4.6.1", "check": "OK", "artifact": "8078909299" }, { "job": 85107013180, "time": 263, "config": "macos-release-x86_64", "r": "4.6.1", "check": "OK", "artifact": "8078811760" }, { "job": 85106791568, "time": 196, "config": "source", "r": "4.6.1", "check": "OK", "artifact": "8078754202" }, { "job": 85107013161, "time": 155, "config": "wasm-release", "r": "4.6.0", "check": "FAIL", "artifact": "" }, { "job": 85107013188, "time": 168, "config": "windows-devel", "r": "4.7.0", "check": "OK", "artifact": "8078776072" }, { "job": 85107013184, "time": 184, "config": "windows-oldrel", "r": "4.5.3", "check": "OK", "artifact": "8078778630" }, { "job": 85107013190, "time": 171, "config": "windows-release", "r": "4.6.1", "check": "OK", "artifact": "8078776467" } ], "_host": "GitHub-Actions", "_buildurl": "https://github.com/r-universe/davzim/actions/runs/28696466726", "_status": "success", "_upstream": "https://github.com/davzim/rtiktoken", "_commit": { "id": "52e7f16a924bdfa255f02670b73452b8af422dcd", "author": "DavZim ", "committer": "DavZim ", "message": "update Readme\n", "time": 1776953214 }, "_maintainer": { "name": "David Zimmermann-Kollenda", "email": "david_j_zimmermann@hotmail.com" }, "_distro": "resolute", "_registered": true, "_dependencies": [ { "package": "R", "version": ">= 4.2", "role": "Depends" }, { "package": "testthat", "version": ">= 3.0.0", "role": "Suggests" } ], "_owner": "davzim", "_selfowned": true, "_usedby": 0, "_updates": [ { "week": "2026-15", "n": 2 }, { "week": "2026-17", "n": 2 } ], "_tags": [], "_topics": [ "bpe", "openai", "rust", "tokenization", "cargo" ], "_stars": 14, "_contributors": [ { "user": "davzim", "count": 27, "uuid": 15910496 }, { "user": "josiahparry", "count": 1, "uuid": 13534508 } ], "_userbio": { "uuid": 15910496, "type": "user", "name": "DavZim", "followers": 71 }, "_downloads": { "count": 259, "source": "https://cranlogs.r-pkg.org/downloads/total/last-month/rtiktoken" }, "_devurl": "https://github.com/davzim/rtiktoken", "_pkgdown": "https://davzim.github.io/rtiktoken/", "_searchresults": 7, "_cargo": true, "_rbuild": "4.6.1", "_assets": [ "extra/citation.cff", "extra/citation.html", "extra/citation.json", "extra/citation.txt", "extra/contents.json", "extra/NEWS.html", "extra/NEWS.txt", "extra/readme.html", "extra/readme.md", "extra/rtiktoken.html", "LICENSE", "manual.pdf" ], "_homeurl": "https://github.com/davzim/rtiktoken", "_realowner": "davzim", "_cranurl": true, "_releases": [ { "version": "0.0.6", "date": "2024-11-06" }, { "version": "0.0.7", "date": "2025-04-14" } ], "_exports": [ "decode_tokens", "get_token_count", "get_tokens", "model_to_tokenizer" ], "_help": [ { "page": "decode_tokens", "title": "Decodes tokens back to text", "topics": [ "decode_tokens" ] }, { "page": "get_token_count", "title": "Returns the number of tokens in a text", "topics": [ "get_token_count" ] }, { "page": "get_tokens", "title": "Converts text to tokens", "topics": [ "get_tokens" ] }, { "page": "model_to_tokenizer", "title": "Gets the name of the tokenizer used by a model", "topics": [ "model_to_tokenizer" ] } ], "_readme": "https://github.com/davzim/rtiktoken/raw/HEAD/README.md", "_rundeps": [], "_score": 4.146128035678238, "_indexed": true, "_nocasepkg": "rtiktoken", "_universes": [ "davzim" ], "_binaries": [ { "r": "4.7.0", "os": "linux", "version": "0.11.0-1", "date": "2026-07-04T05:42:05.000Z", "distro": "resolute", "arch": "aarch64", "commit": "52e7f16a924bdfa255f02670b73452b8af422dcd", "fileid": "https://r2.ropensci.org/a40a75acd32faf30dc60b3429bcbf161742f1e99f643d647b0ea7d80b83e2362", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/28696466726" }, { "r": "4.7.0", "os": "linux", "version": "0.11.0-1", "date": "2026-07-04T05:42:06.000Z", "distro": "resolute", "arch": "x86_64", "commit": "52e7f16a924bdfa255f02670b73452b8af422dcd", "fileid": "https://r2.ropensci.org/551599cc7cc5385aa8eb71773f22bd8eab8df1e4cefd92592f89e42cbe4c45e8", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/28696466726" }, { "r": "4.6.1", "os": "linux", "version": "0.11.0-1", "date": "2026-07-04T05:42:47.000Z", "distro": "resolute", "arch": "aarch64", "commit": "52e7f16a924bdfa255f02670b73452b8af422dcd", "fileid": "https://r2.ropensci.org/af6592ed180c3b9a856110d7bdbea2689a9e1698052cc6a414e87ebdd964c584", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/28696466726" }, { "r": "4.6.1", "os": "linux", "version": "0.11.0-1", "date": "2026-07-04T05:42:04.000Z", "distro": "resolute", "arch": "x86_64", "commit": "52e7f16a924bdfa255f02670b73452b8af422dcd", "fileid": "https://r2.ropensci.org/e19032ac3cc78dbca4655e645048c3bc596934ed1e7a38798b1aee5bdc53bc07", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/28696466726" }, { "r": "4.5.3", "os": "mac", "version": "0.11.0-1", "date": "2026-07-04T05:58:34.000Z", "arch": "aarch64", "commit": "52e7f16a924bdfa255f02670b73452b8af422dcd", "fileid": "https://r2.ropensci.org/202efae8d98b7f23e2c6ac99472a8f6afda7527d7e23f3fb7de1a262feba096c", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/28696466726" }, { "r": "4.5.3", "os": "mac", "version": "0.11.0-1", "date": "2026-07-04T05:43:50.000Z", "arch": "x86_64", "commit": "52e7f16a924bdfa255f02670b73452b8af422dcd", "fileid": "https://r2.ropensci.org/9a768cde87f84a9658cd81fba6eefe6cdc57abfd3078f334ec8c715a6a454c28", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/28696466726" }, { "r": "4.6.1", "os": "mac", "version": "0.11.0-1", "date": "2026-07-04T05:59:30.000Z", "arch": "aarch64", "commit": "52e7f16a924bdfa255f02670b73452b8af422dcd", "fileid": "https://r2.ropensci.org/c3476e0cce2ea5f59c3ab2cc874fd10c88d3b3960066a754789cadf9c05f6b86", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/28696466726" }, { "r": "4.6.1", "os": "mac", "version": "0.11.0-1", "date": "2026-07-04T05:45:50.000Z", "arch": "x86_64", "commit": "52e7f16a924bdfa255f02670b73452b8af422dcd", "fileid": "https://r2.ropensci.org/1c0076b449c652185d454631f90f8635000982c795557dc101f7a6eda60b196c", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/28696466726" }, { "r": "4.7.0", "os": "win", "version": "0.11.0-1", "date": "2026-07-04T05:41:00.000Z", "arch": "x86_64", "commit": "52e7f16a924bdfa255f02670b73452b8af422dcd", "fileid": "https://r2.ropensci.org/028acf38ac127a514e899b8eb70ef7d99b5e43e65e7fa1415973511f801c0a88", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/28696466726" }, { "r": "4.5.3", "os": "win", "version": "0.11.0-1", "date": "2026-07-04T05:41:13.000Z", "arch": "x86_64", "commit": "52e7f16a924bdfa255f02670b73452b8af422dcd", "fileid": "https://r2.ropensci.org/d94cf3b0110831755d30c64c49af7830c55e12ede77ed1ee05e24331aa47b7c6", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/28696466726" }, { "r": "4.6.1", "os": "win", "version": "0.11.0-1", "date": "2026-07-04T05:41:00.000Z", "arch": "x86_64", "commit": "52e7f16a924bdfa255f02670b73452b8af422dcd", "fileid": "https://r2.ropensci.org/b29dead5b6f6102823d16b07e78d3b32f296d75c9d52971b25ccc3f19dcfb311", "status": "success", "check": "OK", "buildurl": "https://github.com/r-universe/davzim/actions/runs/28696466726" } ] }