{"added":"2026-07-05T21:22:17-04:00","apiVersion":1,"behindSince":"2026-07-13","buildDepends":[],"categories":["textproc"],"comment":"Unsupervised text tokenizer for Neural Network-based text generation","compare":"/compare/sentencepiece/index.json","dependentCount":null,"description":"SentencePiece is an unsupervised text tokenizer and detokenizer mainly for\nNeural Network-based text generation systems where the vocabulary size is\npredetermined prior to the neural model training. SentencePiece implements\nsubword units (e.g., byte-pair-encoding (BPE)) and unigram language model\nwith the extension of direct training from raw sentences. SentencePiece\nallows us to make a purely end-to-end system that does not depend on\nlanguage-specific pre/postprocessing.","flavors":[],"html":"https://daemonports.org/midnightbsd/textproc/sentencepiece/","kind":"port","lastUpdate":"2026-07-05T21:22:17-04:00","license":["Apache-2.0"],"maintainer":"ports@MidnightBSD.org","name":"sentencepiece","newest":false,"newestVersion":"0.2.2","noVulnData":true,"origin":"textproc/sentencepiece","otherSystems":[{"maintainer":"yuri@FreeBSD.org","newest":true,"origin":"textproc/sentencepiece","path":"/textproc/sentencepiece/","system":"FreeBSD","updated":"2026-07-13","version":"0.2.2"},{"behindSince":"2026-07-13","origin":"textproc/sentencepiece","path":"/netbsd/textproc/sentencepiece/","system":"NetBSD","version":"0.2.1"},{"behindSince":"2026-07-05","maintainer":"yuri@FreeBSD.org","origin":"textproc/sentencepiece","path":"/dragonfly/textproc/sentencepiece/","system":"DragonFly","version":"0.2.0_1"},{"maintainer":"yuri@FreeBSD.org","newest":true,"noVulnData":true,"origin":"textproc/sentencepiece","path":"","system":"HardenedBSD","version":"0.2.2"},{"maintainer":"yuri@FreeBSD.org","newest":true,"noVulnData":true,"origin":"textproc/sentencepiece","path":"","system":"GhostBSD","version":"0.2.2"}],"packages":null,"pkgName":"sentencepiece-0.2.1","related":["textproc/py-sentencepiece"],"repository":"mports","requiredBy":["textproc/py-sentencepiece"],"requiredByCount":1,"runDepends":[],"status":null,"system":"midnightbsd","systemLabel":"MidnightBSD","version":"0.2.1","versions":[{"commit":"8af06753b2eaf0c58214302c2a501feadc09477a","date":"2026-07-05","version":"0.2.1"}],"vulnerabilities":null,"www":"https://github.com/google/sentencepiece"}