{"added":null,"apiVersion":1,"behindSince":"2026-11-10","buildDepends":[],"categories":["textproc"],"comment":"Unsupervised text tokenizer for Neural Network-based text generation","compare":"/compare/py312-sentencepiece/index.json","dependentCount":null,"description":"SentencePiece is an unsupervised text tokenizer and detokenizer mainly for\nNeural Network-based text generation systems where the vocabulary size is\npredetermined prior to the neural model training. SentencePiece implements\nsubword units (e.g., byte-pair-encoding (BPE)) and unigram language model\nwith the extension of direct training from raw sentences. SentencePiece\nallows us to make a purely end-to-end system that does not depend on\nlanguage-specific pre/postprocessing.","flavors":[],"html":"https://daemonports.org/dragonfly/textproc/py-sentencepiece/","kind":"port","lastUpdate":null,"license":["APACHE20"],"licenses":["Apache-2.0"],"maintainer":"yuri@FreeBSD.org","name":"py312-sentencepiece","newest":false,"newestVersion":"0.2.3","noVulnData":false,"origin":"textproc/py-sentencepiece","otherSystems":[{"maintainer":"yuri@FreeBSD.org","newest":true,"origin":"textproc/py-sentencepiece","path":"/textproc/py-sentencepiece/","system":"FreeBSD","updated":"2026-11-10","version":"0.2.3"},{"behindSince":"2026-07-13","origin":"textproc/py-sentencepiece","path":"/netbsd/textproc/py-sentencepiece/","system":"NetBSD","version":"0.2.1"},{"behindSince":"2026-07-13","maintainer":"ports@MidnightBSD.org","noVulnData":true,"origin":"textproc/py-sentencepiece","path":"/midnightbsd/textproc/py-sentencepiece/","system":"MidnightBSD","updated":"2026-11-04","version":"0.2.1_1"},{"behindSince":"2026-11-10","maintainer":"yuri@FreeBSD.org","noVulnData":true,"origin":"textproc/py-sentencepiece","path":"","system":"HardenedBSD","version":"0.2.2"},{"behindSince":"2026-11-10","maintainer":"yuri@FreeBSD.org","noVulnData":true,"origin":"textproc/py-sentencepiece","path":"","system":"GhostBSD","version":"0.2.2"}],"packages":null,"pkgName":"py312-sentencepiece-0.2.2","related":["textproc/sentencepiece"],"repository":"DPorts","requiredBy":["misc/py-gguf","misc/py-google-genai","misc/py-mistral-common"],"requiredByCount":3,"runDepends":["lang/python312","textproc/sentencepiece"],"status":null,"system":"dragonfly","systemLabel":"DragonFly","version":"0.2.2","versions":[{"date":"2026-11-11","version":"0.2.2"}],"vulnerabilities":null,"www":"https://github.com/google/sentencepiece"}