{"entities":{"Q89656":{"pageid":90944,"ns":120,"title":"Item:Q89656","lastrevid":105566441,"modified":"2026-10-10T12:34:34Z","type":"item","id":"Q89656","labels":{"en":{"language":"en","value":"sentencepiece"}},"descriptions":{"en":{"language":"en","value":"Text Tokenization using Byte Pair Encoding and Unigram Modelling"}},"aliases":{},"claims":{"P31":[{"mainsnak":{"snaktype":"value","property":"P31","hash":"438d0eef0223375d3a92def8ffc6a10a8bf1ca95","datavalue":{"value":{"entity-type":"item","numeric-id":57080,"id":"Q57080"},"type":"wikibase-entityid"},"datatype":"wikibase-item"},"type":"statement","id":"Q89656$FCD7EF40-4EC0-4D6A-AAF8-6626E7113CB7","rank":"normal"}],"P132":[{"mainsnak":{"snaktype":"value","property":"P132","hash":"2595e52c474e14ac824703452b151b52c0834700","datavalue":{"value":"0.2.3","type":"string"},"datatype":"string"},"type":"statement","qualifiers":{"P28":[{"snaktype":"value","property":"P28","hash":"e0001f1844569a4f3bd6cf94da2047c6270ec2b6","datavalue":{"value":{"time":"+2022-11-13T00:00:00Z","timezone":0,"before":0,"after":0,"precision":11,"calendarmodel":"http://www.wikidata.org/entity/Q1985727"},"type":"time"},"datatype":"time"}]},"qualifiers-order":["P28"],"id":"Q89656$6F50CCE2-0687-4D81-BBA1-90DE18776236","rank":"normal"},{"mainsnak":{"snaktype":"value","property":"P132","hash":"b542975f358e64266fc8bf3287bc018a1e05a763","datavalue":{"value":"PACKAGES.rds","type":"string"},"datatype":"string"},"type":"statement","qualifiers":{"P28":[{"snaktype":"value","property":"P28","hash":"bdab782bc09c5c1d8687a0515dd3ffbe9dda94be","datavalue":{"value":{"time":"+2026-07-09T00:00:00Z","timezone":0,"before":0,"after":0,"precision":11,"calendarmodel":"http://www.wikidata.org/entity/Q1985727"},"type":"time"},"datatype":"time"}]},"qualifiers-order":["P28"],"id":"Q89656$F13EB812-C7EC-4199-AB00-61B8A1D4344B","rank":"normal"},{"mainsnak":{"snaktype":"value","property":"P132","hash":"dd764dabaf036ec40584882b7f9127cdfd73d125","datavalue":{"value":"0.1.1","type":"string"},"datatype":"string"},"type":"statement","qualifiers":{"P28":[{"snaktype":"value","property":"P28","hash":"84fdf3ed3ab7383ca4f016ac37ce5ef0ef7cb738","datavalue":{"value":{"time":"+2020-06-04T00:00:00Z","timezone":0,"before":0,"after":0,"precision":11,"calendarmodel":"http://www.wikidata.org/entity/Q1985727"},"type":"time"},"datatype":"time"}]},"qualifiers-order":["P28"],"id":"Q89656$EC558E30-02F4-49C4-BFA8-19A90FB0C1FD","rank":"normal"},{"mainsnak":{"snaktype":"value","property":"P132","hash":"8e566d59c9c8e44ecba7cb45a73aaa454a64a603","datavalue":{"value":"0.1.2","type":"string"},"datatype":"string"},"type":"statement","qualifiers":{"P28":[{"snaktype":"value","property":"P28","hash":"10dbc037f267f73cd366302adbadee76657b2209","datavalue":{"value":{"time":"+2020-06-08T00:00:00Z","timezone":0,"before":0,"after":0,"precision":11,"calendarmodel":"http://www.wikidata.org/entity/Q1985727"},"type":"time"},"datatype":"time"}]},"qualifiers-order":["P28"],"id":"Q89656$1704AB27-784B-49EC-BB81-2125439B75A4","rank":"normal"},{"mainsnak":{"snaktype":"value","property":"P132","hash":"ca91462f2866604b3ca8bbc61cec6d3ec67a5aeb","datavalue":{"value":"0.2.1","type":"string"},"datatype":"string"},"type":"statement","qualifiers":{"P28":[{"snaktype":"value","property":"P28","hash":"284cb704c1ef14a8cc66c4147b73bf6c18c95c07","datavalue":{"value":{"time":"+2021-12-21T00:00:00Z","timezone":0,"before":0,"after":0,"precision":11,"calendarmodel":"http://www.wikidata.org/entity/Q1985727"},"type":"time"},"datatype":"time"}]},"qualifiers-order":["P28"],"id":"Q89656$F1E5B630-24C3-487E-97A3-C430AF7151A1","rank":"normal"},{"mainsnak":{"snaktype":"value","property":"P132","hash":"19de1ab48eb22f13f7a49f574a021ffab63cbc7c","datavalue":{"value":"0.2.2","type":"string"},"datatype":"string"},"type":"statement","qualifiers":{"P28":[{"snaktype":"value","property":"P28","hash":"1a54434e20074f3f6b59987a5d81b4852d3c35ec","datavalue":{"value":{"time":"+2022-11-09T00:00:00Z","timezone":0,"before":0,"after":0,"precision":11,"calendarmodel":"http://www.wikidata.org/entity/Q1985727"},"type":"time"},"datatype":"time"}]},"qualifiers-order":["P28"],"id":"Q89656$38491B61-09CF-4F0F-9F6C-3EC2E4E0AA3B","rank":"normal"},{"mainsnak":{"snaktype":"value","property":"P132","hash":"37ddd2aff68fe722f88fecf3f4c8c98f03e76c31","datavalue":{"value":"0.2.4","type":"string"},"datatype":"string"},"type":"statement","qualifiers":{"P28":[{"snaktype":"value","property":"P28","hash":"fc5d2bbf40c46bc96accd37983b80b5530492556","datavalue":{"value":{"time":"+2025-11-27T00:00:00Z","timezone":0,"before":0,"after":0,"precision":11,"calendarmodel":"http://www.wikidata.org/entity/Q1985727"},"type":"time"},"datatype":"time"}]},"qualifiers-order":["P28"],"id":"Q89656$B2AE9FFB-8CB3-4F9D-AB75-DE7342DACA1E","rank":"normal"},{"mainsnak":{"snaktype":"value","property":"P132","hash":"46bbeb687c37ca8b18cce891ac55fb67a6a23de8","datavalue":{"value":"0.2","type":"string"},"datatype":"string"},"type":"statement","qualifiers":{"P28":[{"snaktype":"value","property":"P28","hash":"6e0bac8f0fcdf97bdc17562f0f99c46eeb852b29","datavalue":{"value":{"time":"+2021-12-15T00:00:00Z","timezone":0,"before":0,"after":0,"precision":11,"calendarmodel":"http://www.wikidata.org/entity/Q1985727"},"type":"time"},"datatype":"time"}]},"qualifiers-order":["P28"],"id":"Q89656$C8D5349B-2BEC-4124-9FA7-0414D12FDECE","rank":"normal"},{"mainsnak":{"snaktype":"value","property":"P132","hash":"d8ababcc31a736b189512058b0c65a3355fc2446","datavalue":{"value":"0.2.5","type":"string"},"datatype":"string"},"type":"statement","qualifiers":{"P28":[{"snaktype":"value","property":"P28","hash":"36798a9367dd4f65c7b2f0e18c37acb2a3e004d3","datavalue":{"value":{"time":"+2026-02-09T00:00:00Z","timezone":0,"before":0,"after":0,"precision":11,"calendarmodel":"http://www.wikidata.org/entity/Q1985727"},"type":"time"},"datatype":"time"}]},"qualifiers-order":["P28"],"id":"Q89656$9720071C-0114-466D-B03C-9A5BB51606D2","rank":"normal"}],"P16":[{"mainsnak":{"snaktype":"value","property":"P16","hash":"ef4bd0e720ffb64cbfb73f8899fc76255f935df5","datavalue":{"value":{"entity-type":"item","numeric-id":89651,"id":"Q89651"},"type":"wikibase-entityid"},"datatype":"wikibase-item"},"type":"statement","id":"Q89656$42D489AA-976D-4DFA-91DD-F2D5E59F6255","rank":"normal"}],"P19":[{"mainsnak":{"snaktype":"value","property":"P19","hash":"dead37a8cbf5f40599e6dbf4ddf8d1fd0c287dcb","datavalue":{"value":{"entity-type":"item","numeric-id":89651,"id":"Q89651"},"type":"wikibase-entityid"},"datatype":"wikibase-item"},"type":"statement","id":"Q89656$5134B397-87CD-4223-AD39-4060C8E4C0F1","rank":"normal"}],"P163":[{"mainsnak":{"snaktype":"value","property":"P163","hash":"e833327f863f180e0294a5061420c182fb40052a","datavalue":{"value":{"entity-type":"item","numeric-id":56821,"id":"Q56821"},"type":"wikibase-entityid"},"datatype":"wikibase-item"},"type":"statement","id":"Q89656$DF1FC3A9-1749-4312-BD83-150E2D8205EE","rank":"normal"}],"P585":[{"mainsnak":{"snaktype":"value","property":"P585","hash":"24c3e3d95d6aceb19323da214a060d1bb0535ab2","datavalue":{"value":{"entity-type":"item","numeric-id":57178,"id":"Q57178"},"type":"wikibase-entityid"},"datatype":"wikibase-item"},"type":"statement","id":"Q89656$F16F9591-DE36-4A58-BA5F-70E011939B99","rank":"normal"},{"mainsnak":{"snaktype":"value","property":"P585","hash":"d595b9251c15d55b682bbb22e27ac12f91c3ba38","datavalue":{"value":{"entity-type":"item","numeric-id":20394,"id":"Q20394"},"type":"wikibase-entityid"},"datatype":"wikibase-item"},"type":"statement","qualifiers":{"P132":[{"snaktype":"value","property":"P132","hash":"1520c8e2cd338f55d57f9169eb44a375ecafc28d","datavalue":{"value":"\u2265 0.11.5","type":"string"},"datatype":"string"}]},"qualifiers-order":["P132"],"id":"Q89656$E48227D9-AAB0-4741-99B0-3AF83881CD12","rank":"normal"}],"P223":[{"mainsnak":{"snaktype":"value","property":"P223","hash":"189b3e52b542f11ed7152689d978d72eadb24d89","datavalue":{"value":{"entity-type":"item","numeric-id":89655,"id":"Q89655"},"type":"wikibase-entityid"},"datatype":"wikibase-item"},"type":"statement","id":"Q89656$B3DC0541-AE62-49E4-B86B-81102613B1D6","rank":"normal"}],"P229":[{"mainsnak":{"snaktype":"value","property":"P229","hash":"61c5dfe2e5218ee692f5970c405f1a31b23fabe9","datavalue":{"value":"sentencepiece","type":"string"},"datatype":"external-id"},"type":"statement","id":"Q89656$082BD9E8-A308-4EEB-A6FC-BAD8D9FD9DF1","rank":"normal"}],"P114":[{"mainsnak":{"snaktype":"value","property":"P114","hash":"251fd109c2cd8c1ee39b48cd109c9f4b1c9fcf5c","datavalue":{"value":{"entity-type":"item","numeric-id":13519,"id":"Q13519"},"type":"wikibase-entityid"},"datatype":"wikibase-item"},"type":"statement","id":"Q89656$6AA9FCC4-8BEF-4BF1-B3C3-BBFDE4323D33","rank":"normal"}],"P1460":[{"mainsnak":{"snaktype":"value","property":"P1460","hash":"908c3454b3659c4b140ccce33c5aee31081edc8d","datavalue":{"value":{"entity-type":"item","numeric-id":5976450,"id":"Q5976450"},"type":"wikibase-entityid"},"datatype":"wikibase-item"},"type":"statement","id":"Q89656$7B8D5837-6DFB-426D-B3EE-73BE43B6B5C6","rank":"normal"}],"P170":[{"mainsnak":{"snaktype":"value","property":"P170","hash":"b5198d8b7a5a273a85bcd63cbbb94f6682123e8d","datavalue":{"value":{"time":"+2026-02-09T00:00:00Z","timezone":0,"before":0,"after":0,"precision":11,"calendarmodel":"http://www.wikidata.org/entity/Q1985727"},"type":"time"},"datatype":"time"},"type":"statement","id":"Q89656$5EA771C6-BF4E-41FF-B48E-AB807760A568","rank":"normal"}],"P342":[{"mainsnak":{"snaktype":"value","property":"P342","hash":"a0beaece152e3d4cfe5f01b8a4c02caff7e1e6cc","datavalue":{"value":{"entity-type":"item","numeric-id":56774,"id":"Q56774"},"type":"wikibase-entityid"},"datatype":"wikibase-item"},"type":"statement","qualifiers":{"P132":[{"snaktype":"value","property":"P132","hash":"fca10554373f264859696efd898e7d44319afdac","datavalue":{"value":"\u2265 2.10","type":"string"},"datatype":"string"}]},"qualifiers-order":["P132"],"id":"Q89656$181EF915-9942-4487-90A5-CC7F7D73079D","rank":"normal"}],"P1459":[{"mainsnak":{"snaktype":"value","property":"P1459","hash":"5c81ebd2b351675b04d69621b731f444ff33cd06","datavalue":{"value":"Unsupervised text tokenizer allowing to perform byte pair encoding and unigram modelling.     Wraps the 'sentencepiece' library <https://github.com/google/sentencepiece> which provides a language independent tokenizer to split text in words and smaller subword units.     The techniques are explained in the paper \"SentencePiece: A simple and language independent subword tokenizer and detokenizer for Neural Text Processing\" by Taku Kudo and John Richardson (2018) <doi:10.18653/v1/D18-2012>.    Provides as well straightforward access to pretrained byte pair encoding models and subword embeddings trained on Wikipedia using 'word2vec',     as described in \"BPEmb: Tokenization-free Pre-trained Subword Embeddings in 275 Languages\" by Benjamin Heinzerling and Michael Strube (2018) <http://www.lrec-conf.org/proceedings/lrec2018/pdf/1049.pdf>.","type":"string"},"datatype":"string"},"type":"statement","id":"Q89656$9A41EA6C-C6F7-4395-9229-ACB0AF3D8692","rank":"normal"}]},"sitelinks":{"mardi":{"site":"mardi","title":"Sentencepiece","badges":[],"url":"https://portal.mardi4nfdi.de/wiki/Sentencepiece"}}}}}