{"added":null,"apiVersion":1,"behindSince":null,"buildDepends":[],"categories":["python","textproc"],"comment":"Tokenizer for Icelandic text","compare":null,"dependentCount":null,"description":"Tokenizer: A tokenizer for Icelandic text\n\nTokenization is a necessary first step in many natural language processing\ntasks, such as word counting, parsing, spell checking, corpus generation, and\nstatistical analysis of text.","flavors":[],"html":"https://daemonports.org/dragonfly/textproc/py-tokenizer/","kind":"port","lastUpdate":null,"license":["MIT"],"licenses":["MIT"],"maintainer":"otis@FreeBSD.org","name":"py312-tokenizer","newest":true,"newestVersion":"3.6.4","noVulnData":false,"origin":"textproc/py-tokenizer","otherSystems":[{"maintainer":"otis@FreeBSD.org","newest":true,"origin":"textproc/py-tokenizer","path":"/textproc/py-tokenizer/","system":"FreeBSD","updated":"2026-07-12","version":"3.6.4"},{"maintainer":"otis@FreeBSD.org","newest":true,"noVulnData":true,"origin":"textproc/py-tokenizer","path":"","system":"HardenedBSD","version":"3.6.4"},{"maintainer":"otis@FreeBSD.org","newest":true,"noVulnData":true,"origin":"textproc/py-tokenizer","path":"","system":"GhostBSD","version":"3.6.4"}],"packages":null,"pkgName":"py312-tokenizer-3.6.4","related":null,"repository":"DPorts","requiredBy":[],"requiredByCount":0,"runDepends":["lang/python312"],"status":null,"system":"dragonfly","systemLabel":"DragonFly","version":"3.6.4","versions":[{"date":"2026-11-11","version":"3.6.4"}],"vulnerabilities":null,"www":"https://github.com/mideind/Tokenizer"}