{"_id":"strtok2","_rev":"4211023","name":"strtok2","description":"A streaming tokenizer","dist-tags":{"latest":"1.2.1"},"maintainers":[{"name":"borewit","email":"borewit@xs4all.nl"}],"time":{"modified":"2026-03-04T17:05:55.000Z","created":"2013-09-04T11:04:09.061Z","1.2.1":"2017-09-17T10:57:50.853Z","1.2.0":"2017-06-30T07:06:05.669Z","1.1.1":"2017-06-18T20:35:24.045Z","1.0.4":"2016-12-02T18:58:36.797Z","1.0.3":"2016-11-30T22:04:44.718Z","1.0.2":"2016-11-30T19:33:08.785Z","1.0.1":"2014-01-13T05:47:39.637Z","1.0.0":"2013-09-04T11:26:37.724Z","0.2.0":"2013-09-04T11:04:09.061Z"},"users":{},"author":{"name":"Borewit","url":"https://github.com/Borewit"},"repository":{"type":"git","url":"git+https://github.com/Borewit/strtok2.git"},"versions":{"1.2.1":{"name":"strtok2","version":"1.2.1","description":"A streaming tokenizer","author":{"name":"Borewit","url":"https://github.com/Borewit"},"contributors":[{"name":"Peter Griess","email":"pg@std.in","url":"https://github.com/pgriess"},{"name":"Andrew Kelley","email":"superjoe30@gmail.com","url":"https://github.com/andrewrk"}],"scripts":{"compile":"tsc","clean":"gulp clean","prepare":"gulp","mocha":"mocha --require ts-node/register test","test":"gulp test"},"engines":{"node":">=0.1.98"},"repository":{"type":"git","url":"git+https://github.com/Borewit/strtok2.git"},"license":"BSD-3-Clause","main":"./lib/index.js","typings":"lib/index","bugs":{"url":"https://github.com/Borewit/strtok2/issues"},"devDependencies":{"@types/chai":"^4.0.4","@types/mocha":"^2.2.43","@types/node":"^7.0.43","chai":"^4.1.2","debug":"^2.6.8","del":"^3.0.0","gulp":"^3.9.1","gulp-clean":"^0.3.2","gulp-mocha":"^4.3.1","gulp-tsc":"^1.3.2","mocha":"^3.5.3","ts-node":"^3.3.0","typescript":"^2.5.2","typings":"^2.1.1"},"dependencies":{"token-types":"^0.1.2"},"keywords":["tokenizer","promise","parser","binary","endian","integer","uint","token","read","stream","streaming"],"gitHead":"884d8c771e23f06ed4097ef748fb8c20bc86f877","homepage":"https://github.com/Borewit/strtok2#readme","_id":"strtok2@1.2.1","_npmVersion":"5.2.0","_nodeVersion":"4.8.4","_npmUser":{"name":"borewit","email":"borewit@xs4all.nl"},"dist":{"shasum":"7ebe29f7fb3299fb8c1172541696dbca20ca864c","size":12365,"noattachment":false,"key":"/strtok2/-/strtok2-1.2.1.tgz","tarball":"http://registry.cnpm.dingdandao.com/strtok2/download/strtok2-1.2.1.tgz"},"maintainers":[{"name":"borewit","email":"borewit@xs4all.nl"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/strtok2-1.2.1.tgz_1505645869868_0.3665791149251163"},"directories":{},"publish_time":1505645870853,"_hasShrinkwrap":false,"deprecated":"Use music-metadata based on strtok3 instead!","_cnpm_publish_time":1505645870853,"_cnpmcore_publish_time":"2021-12-16T18:28:27.365Z"},"1.2.0":{"name":"strtok2","version":"1.2.0","description":"A streaming tokenizer","author":{"name":"Borewit","url":"https://github.com/Borewit"},"contributors":[{"name":"Peter Griess","email":"pg@std.in","url":"https://github.com/pgriess"},{"name":"Andrew Kelley","email":"superjoe30@gmail.com","url":"https://github.com/andrewrk"}],"scripts":{"compile":"tsc","clean":"gulp clean","prepare":"gulp","mocha":"mocha --require ts-node/register test","test":"gulp test"},"engines":{"node":">=0.1.98"},"repository":{"type":"git","url":"git+https://github.com/Borewit/strtok2.git"},"license":"BSD-3-Clause","main":"./lib/index.js","typings":"lib/index","bugs":{"url":"https://github.com/Borewit/strtok2/issues"},"devDependencies":{"@types/chai":"^4.0.0","@types/mocha":"^2.2.41","@types/node":"^7.0.31","chai":"^4.0.2","debug":"^2.6.8","del":"^3.0.0","gulp":"^3.9.1","gulp-clean":"^0.3.2","gulp-mocha":"^4.3.1","gulp-tsc":"^1.3.2","mocha":"^3.4.2","ts-node":"^3.0.6","typescript":"^2.3.4","typings":"^2.1.1"},"dependencies":{"token-types":"0.0.2"},"keywords":["tokenizer","promise","parser","binary","endian","integer","uint","token","read","stream","streaming"],"gitHead":"d18c0f3d8cd7049c5856442489625cbe5496334d","homepage":"https://github.com/Borewit/strtok2#readme","_id":"strtok2@1.2.0","_shasum":"b329b83ea25aa9dccc9c7429360373e196e6e72f","_from":".","_npmVersion":"3.10.9","_nodeVersion":"4.8.0","_npmUser":{"name":"borewit","email":"borewit@xs4all.nl"},"dist":{"shasum":"b329b83ea25aa9dccc9c7429360373e196e6e72f","size":12304,"noattachment":false,"key":"/strtok2/-/strtok2-1.2.0.tgz","tarball":"http://registry.cnpm.dingdandao.com/strtok2/download/strtok2-1.2.0.tgz"},"maintainers":[{"name":"borewit","email":"borewit@xs4all.nl"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/strtok2-1.2.0.tgz_1498806364557_0.7276717007625848"},"directories":{},"publish_time":1498806365669,"_hasShrinkwrap":false,"deprecated":"Use music-metadata based on strtok3 instead!","_cnpm_publish_time":1498806365669,"_cnpmcore_publish_time":"2021-12-16T18:28:27.578Z"},"1.1.1":{"name":"strtok2","version":"1.1.1","description":"A streaming tokenizer","author":{"name":"Borewit","url":"https://github.com/Borewit"},"contributors":[{"name":"Peter Griess","email":"pg@std.in","url":"https://github.com/pgriess"},{"name":"Andrew Kelley","email":"superjoe30@gmail.com","url":"https://github.com/andrewrk"}],"scripts":{"compile":"tsc","test":"mocha --require ts-node/register test","lint":"tslint --format verbose lib/*.ts","dist":"copyfiles -f src/*.js src/*.js.map src/*.d.ts lib"},"engines":{"node":">=0.1.98"},"repository":{"type":"git","url":"git+https://github.com/Borewit/strtok2.git"},"license":"BSD-3-Clause","main":"./lib/index.js","typings":"lib/index","bugs":{"url":"https://github.com/Borewit/strtok2/issues"},"devDependencies":{"@types/chai":"^4.0.0","@types/mocha":"^2.2.41","@types/node":"^7.0.31","chai":"^4.0.2","copyfiles":"^1.2.0","mocha":"^3.4.2","ts-node":"^3.0.6","typescript":"^2.3.4","typings":"^2.1.1"},"gitHead":"e86ad543c7bacda7d7b7ecc6a36943851e0a8c71","homepage":"https://github.com/Borewit/strtok2#readme","_id":"strtok2@1.1.1","_shasum":"0b8dc2b3930924794a267a92d20417fee3cf7297","_from":".","_npmVersion":"3.10.9","_nodeVersion":"4.7.2","_npmUser":{"name":"borewit","email":"borewit@xs4all.nl"},"dist":{"shasum":"0b8dc2b3930924794a267a92d20417fee3cf7297","size":18348,"noattachment":false,"key":"/strtok2/-/strtok2-1.1.1.tgz","tarball":"http://registry.cnpm.dingdandao.com/strtok2/download/strtok2-1.1.1.tgz"},"maintainers":[{"name":"borewit","email":"borewit@xs4all.nl"}],"_npmOperationalInternal":{"host":"s3://npm-registry-packages","tmp":"tmp/strtok2-1.1.1.tgz_1497818122847_0.4013492742087692"},"directories":{},"publish_time":1497818124045,"_hasShrinkwrap":false,"deprecated":"Use music-metadata based on strtok3 instead!","_cnpm_publish_time":1497818124045,"_cnpmcore_publish_time":"2021-12-16T18:28:27.922Z"},"1.0.4":{"name":"strtok2","version":"1.0.4","description":"A streaming tokenizer","author":{"name":"Borewit","url":"https://github.com/Borewit"},"contributors":[{"name":"Peter Griess","email":"pg@std.in","url":"https://github.com/pgriess"},{"name":"Andrew Kelley","email":"superjoe30@gmail.com","url":"https://github.com/andrewrk"}],"scripts":{"test":"find test/ -name 'test-*.js' -exec node {} \\;"},"engines":{"node":">=0.1.98"},"repository":{"type":"git","url":"git+https://github.com/Borewit/strtok2.git"},"license":"BSD-3-Clause","main":"./lib/index.js","typings":"lib/index","bugs":{"url":"https://github.com/Borewit/strtok2/issues"},"gitHead":"f51b5b06014b81233d22c563a329b865cb9d4f6e","homepage":"https://github.com/Borewit/strtok2#readme","_id":"strtok2@1.0.4","_shasum":"df9ffb2e70a3336b4660e267ef0547e610275fc1","_from":".","_npmVersion":"3.10.9","_nodeVersion":"4.4.2","_npmUser":{"name":"borewit","email":"borewit@xs4all.nl"},"dist":{"shasum":"df9ffb2e70a3336b4660e267ef0547e610275fc1","size":16594,"noattachment":false,"key":"/strtok2/-/strtok2-1.0.4.tgz","tarball":"http://registry.cnpm.dingdandao.com/strtok2/download/strtok2-1.0.4.tgz"},"maintainers":[{"name":"borewit","email":"borewit@xs4all.nl"}],"_npmOperationalInternal":{"host":"packages-18-east.internal.npmjs.com","tmp":"tmp/strtok2-1.0.4.tgz_1480705116030_0.9712514437269419"},"directories":{},"publish_time":1480705116797,"_hasShrinkwrap":false,"deprecated":"depricated, use strtok3 & music-metadata instead","_cnpm_publish_time":1480705116797,"_cnpmcore_publish_time":"2021-12-16T18:28:28.171Z"},"1.0.3":{"name":"strtok2","version":"1.0.3","description":"A streaming tokenizer","author":{"name":"Borewit","url":"https://github.com/Borewit"},"contributors":[{"name":"Peter Griess","email":"pg@std.in","url":"https://github.com/pgriess"},{"name":"Andrew Kelley","email":"superjoe30@gmail.com","url":"https://github.com/andrewrk"}],"scripts":{"test":"find test/ -name 'test-*.js' -exec node {} \\;"},"engines":{"node":">=0.1.98"},"repository":{"type":"git","url":"git+https://github.com/Borewit/strtok2.git"},"license":"BSD","main":"./lib/index.js","typings":"lib/index","bugs":{"url":"https://github.com/Borewit/strtok2/issues"},"gitHead":"e6ae23067354eb0554742fec45cd070bb385cb51","homepage":"https://github.com/Borewit/strtok2#readme","_id":"strtok2@1.0.3","_shasum":"e76fb45c4f95e525b9f62ecbbd1386e51f26366e","_from":".","_npmVersion":"3.10.9","_nodeVersion":"4.4.2","_npmUser":{"name":"borewit","email":"borewit@xs4all.nl"},"dist":{"shasum":"e76fb45c4f95e525b9f62ecbbd1386e51f26366e","size":15757,"noattachment":false,"key":"/strtok2/-/strtok2-1.0.3.tgz","tarball":"http://registry.cnpm.dingdandao.com/strtok2/download/strtok2-1.0.3.tgz"},"maintainers":[{"name":"borewit","email":"borewit@xs4all.nl"}],"_npmOperationalInternal":{"host":"packages-12-west.internal.npmjs.com","tmp":"tmp/strtok2-1.0.3.tgz_1480543482522_0.2971975211985409"},"directories":{},"publish_time":1480543484718,"_hasShrinkwrap":false,"deprecated":"depricated, use strtok3 & music-metadata instead","_cnpm_publish_time":1480543484718,"_cnpmcore_publish_time":"2021-12-16T18:28:28.469Z"},"1.0.2":{"name":"strtok2","version":"1.0.2","description":"A streaming tokenizer","author":{"name":"Borewit","url":"https://github.com/Borewit"},"contributors":[{"name":"Peter Griess","email":"pg@std.in","url":"https://github.com/pgriess"},{"name":"Andrew Kelley","email":"superjoe30@gmail.com","url":"https://github.com/andrewrk"}],"scripts":{"test":"find test/ -name 'test-*.js' -exec node {} \\;"},"engines":{"node":">=0.1.98"},"repository":{"type":"git","url":"git+https://github.com/Borewit/strtok2.git"},"license":"BSD","main":"./lib/index.js","typings":"lib/index","bugs":{"url":"https://github.com/Borewit/strtok2/issues"},"gitHead":"fdfcf1d66c681989b396fd8c16653884149d4fe9","homepage":"https://github.com/Borewit/strtok2#readme","_id":"strtok2@1.0.2","_shasum":"285408e2ffea6dad120443d5841650c9c92f6527","_from":".","_npmVersion":"3.10.9","_nodeVersion":"4.4.2","_npmUser":{"name":"borewit","email":"borewit@xs4all.nl"},"dist":{"shasum":"285408e2ffea6dad120443d5841650c9c92f6527","size":15781,"noattachment":false,"key":"/strtok2/-/strtok2-1.0.2.tgz","tarball":"http://registry.cnpm.dingdandao.com/strtok2/download/strtok2-1.0.2.tgz"},"maintainers":[{"name":"borewit","email":"borewit@xs4all.nl"}],"_npmOperationalInternal":{"host":"packages-18-east.internal.npmjs.com","tmp":"tmp/strtok2-1.0.2.tgz_1480534387984_0.7075886772945523"},"directories":{},"publish_time":1480534388785,"_hasShrinkwrap":false,"deprecated":"depricated, use strtok3 & music-metadata instead","_cnpm_publish_time":1480534388785,"_cnpmcore_publish_time":"2021-12-16T18:28:28.732Z"},"1.0.1":{"name":"strtok2","version":"1.0.1","description":"A streaming tokenizer","author":{"name":"Peter Griess","email":"pg@std.in"},"scripts":{"test":"find test/ -name 'test-*.js' -exec node {} \\;"},"engines":{"node":">=0.1.98"},"repository":{"type":"git","url":"http://github.com/pgriess/node-strtok"},"license":{"type":"BSD","url":"http://github.com/pgriess/node-strtok/blob/master/LICENSE"},"main":"./lib/strtok.js","readmeFilename":"README.md","bugs":{"url":"https://github.com/pgriess/node-strtok/issues"},"homepage":"https://github.com/pgriess/node-strtok","_id":"strtok2@1.0.1","dist":{"shasum":"cec2ca202154c6e46ee04e1dbf55dd5b71152cfb","size":14319,"noattachment":false,"key":"/strtok2/-/strtok2-1.0.1.tgz","tarball":"http://registry.cnpm.dingdandao.com/strtok2/download/strtok2-1.0.1.tgz"},"_from":".","_npmVersion":"1.3.21","_npmUser":{"name":"superjoe","email":"superjoe30@gmail.com"},"maintainers":[{"name":"borewit","email":"borewit@xs4all.nl"}],"directories":{},"publish_time":1389592059637,"_hasShrinkwrap":false,"deprecated":"depricated, use strtok3 & music-metadata instead","_cnpm_publish_time":1389592059637,"_cnpmcore_publish_time":"2021-12-16T18:28:28.941Z"},"1.0.0":{"name":"strtok2","version":"1.0.0","description":"A streaming tokenizer","author":{"name":"Peter Griess","email":"pg@std.in"},"scripts":{"test":"find test/ -name 'test-*.js' -exec node {} \\;"},"engines":{"node":">=0.1.98"},"repository":{"type":"git","url":"http://github.com/pgriess/node-strtok"},"license":{"type":"BSD","url":"http://github.com/pgriess/node-strtok/blob/master/LICENSE"},"main":"./lib/strtok.js","readmeFilename":"README.md","bugs":{"url":"https://github.com/pgriess/node-strtok/issues"},"_id":"strtok2@1.0.0","dist":{"shasum":"8a51093bd1c37ff183dea497d6b9a17eacbb37b1","size":14331,"noattachment":false,"key":"/strtok2/-/strtok2-1.0.0.tgz","tarball":"http://registry.cnpm.dingdandao.com/strtok2/download/strtok2-1.0.0.tgz"},"_from":".","_npmVersion":"1.3.8","_npmUser":{"name":"superjoe","email":"superjoe30@gmail.com"},"maintainers":[{"name":"borewit","email":"borewit@xs4all.nl"}],"directories":{},"publish_time":1378293997724,"_hasShrinkwrap":false,"deprecated":"depricated, use strtok3 & music-metadata instead","_cnpm_publish_time":1378293997724,"_cnpmcore_publish_time":"2021-12-16T18:28:29.144Z"},"0.2.0":{"name":"strtok2","version":"0.2.0","description":"A streaming tokenizer","author":{"name":"Peter Griess","email":"pg@std.in"},"scripts":{"test":"find test/ -name 'test-*.js' -exec node {} \\;"},"engines":{"node":">=0.1.98"},"repository":{"type":"git","url":"http://github.com/pgriess/node-strtok"},"license":{"type":"BSD","url":"http://github.com/pgriess/node-strtok/blob/master/LICENSE"},"main":"./lib/strtok.js","readmeFilename":"README.md","bugs":{"url":"https://github.com/pgriess/node-strtok/issues"},"_id":"strtok2@0.2.0","dist":{"shasum":"d9b75b0980bba8a8011538be01e1aac715b32071","size":14300,"noattachment":false,"key":"/strtok2/-/strtok2-0.2.0.tgz","tarball":"http://registry.cnpm.dingdandao.com/strtok2/download/strtok2-0.2.0.tgz"},"_from":".","_npmVersion":"1.3.8","_npmUser":{"name":"superjoe","email":"superjoe30@gmail.com"},"maintainers":[{"name":"borewit","email":"borewit@xs4all.nl"}],"directories":{},"publish_time":1378292649061,"_hasShrinkwrap":false,"deprecated":"this version has been deprecated","_cnpm_publish_time":1378292649061,"_cnpmcore_publish_time":"2021-12-16T18:28:29.344Z"}},"readme":"[![Build Status](https://travis-ci.org/Borewit/strtok2.svg?branch=master)](https://travis-ci.org/Borewit/strtok2)\n[![NPM version](https://badge.fury.io/js/strtok2.svg)](https://npmjs.org/package/strtok2)\n[![npm downloads](http://img.shields.io/npm/dm/strtok2.svg)](https://npmjs.org/package/strtok2)\n[![Dependencies](https://david-dm.org/Borewit/strtok2.svg)](https://david-dm.org/Borewit/strtok2)\n\nA streaming tokenizer for [NodeJS](http://nodejs.org).\n\nSee also **strtok3** on [github.com](https://github.com/Borewit/strtok3) or [npmjs.org](https://npmjs.org/package/strtok3)\n\nParsing data coming off the wire in an event-driven environment can be a\ndifficult proposition, with naive implementations buffering all received data\nin memory until a message has been received in its entirety. Not only is this\ninefficient from a memory standpoint, but it may not be possible to determine\nthe that a message has been fully received without attempting to parse it.\nThis requires a parser that can gracefully handle incomplete messages and\npick up where it left off. To make this task easier, `node-strtok` provides\n\n* Tokenizing primitives for common network datatypes (e.g. signed and\n  unsigned integers in various endian-nesses).\n* A callback-driven approach well suited to an asynchronous environment (e.g.\n  to allow the application to asynchronously ask another party for\n  information about what the next type should be)\n* An easily extensible type system for adding support for new,\n  application-defined types to the core.\n* Very good performance; the built-in MsgPack parser performs within 30% of\n  the native C++ implementation.\n\n## Usage\n\nThe `strtok2` library has only one method: `parse()`.  This method takes a\n`net.Stream` (really any `EventEmitter` that pumps out `data` events) and a\ncallback, which is invoked when a complete token has been read from the stream.\nThe callback takes a single argument: the token just read from the stream, and\nis expected to return the type of token to read from the stream next (e.g.\n`strtok.UINT32_BE`). It is this callback that ultimately implements the\napplication protocol, consuming the provided tokens and instructing\n`node-strtok` the type of the next token to read. It is this inverted control\nflow that allows `node-strtok` to function efficiently as an interruptable\nparser well suited to NodeJS.\n\nThe meat of implementing an application protocol is to be found in what it does\nin its callback function.\n\nWhen `parse()` is invoked, it immediately invokes the callback with a value of\n`undefined`. In this way, the application can indicate the type of token to\nread from the stream first. For example, the MsgPack implementation always\nreads a `strtok.UINT8` when it doesn't know what's coming next; it interprets\nthe resulting value as a type (e.g. integer, array, etc) to determine what type\nof value is coming next.\n\nImplicit in this model is that the callback itself must maintain all\nprotocol-specific state on its own. Often this will include what type of\nstructure it's attmpting to parse from the stream, and the progress made so far\nin parsing that structure.\n\n### Tokens\n\n`node-strtok` supports a wide variety of numerical tokens out of the box:\n\n* `UINT8`\n* `UINT16_BE`, `UINT16_LE`\n* `UINT24_BE`, `UINT24_LE`\n* `UINT32_BE`, `UINT32_LE`\n* `INT8`\n* `INT16_BE`, `INT16_LE`\n* `INT24_BE`, `INT24_LE`\n* `INT32_BE`\n\nTokens are derived from: [token-types](https://github.com/Borewit/token-types).\n\n#### Special tokens\n\nThere are a handful of \"special\" tokens which have special meaning when\nreturned from the protocol callback: `DONE` and `DEFER`. \n\nThe `DONE` token indicates to `strtok.parse()` that the protocol parsing loop\nhas come to an end, and that no more data need be read off of the stream. This\ncauses `strtok.parse()` to disengage from the stream. The callback will not be\ninvoked again. In addition, upon receiving `DONE`, `strtok.parse()` may have\nexcess data buffers which it has pulled off of the stream, but which it did not\nconsume while being directed by the protocol callback. Rather than dropping\nthis data on the floor, `strtok.parse()` will synthesize and emit `data` events\non the stream which it was operating on. The idea is to facilitate protocol\nstacks which want to use `node-strtok` to parse some portion of the protocol\nand then hand off processing to some other codepath(s). In this case, it is\nexpected that the protocol callback will add `data` event listeners to the\nstream immediately before returning `DONE`.\n\nThe `DEFER` token indicates that the protocol doesn't know what type of token\nto read from the stream next. Perhaps the protocol needs to consult some\nout-of-process data-structure, or wait for some other event to occur. To support\nthis case, the protocol callback is actually invoked with 2 arguments: the\nvalue and a defer callback. It is this second parameter, a callback, that must\nbe invoked with the desired token type once the protocol layer has figured this\nout. Note that between the time `DEFER` is returned and the callback is\ninvoked, `strtok.parse()` is buffering all data received from the stream.\n\n#### Complex tokens\n\nThe token types returned from the protocol callback are simply objects with\n1) a `get()` method that takes a `Buffer` and an offset and returns the token\nvalue, and 2) a `len` field that indicates the number of bytes to consume.\nAny JavaScript object that meets this criteria can be returned from the\nprotocol callback. `node-strtok` ships with two built-in types for supporting\ncommon behavior that take advantage of this\n\n* `BufferType` -- consume a fixed number of bytes from the stream and\n  return a `Buffer` instance containing these bytes.\n* `StringType` -- consume a fixed number of bytes from the stream and\n  return a string with a specified encoding.\n\nImplementing such types is extremely simple. The `BufferType` implementation\nis given below:\n\n```javascript\nvar BufferType = function(l) {\n    var self = this;\n\n    self.len = l;\n\n    self.get = function(buf, off) {\n        return buf.slice(off, off + this.len);\n    };\n};\nexports.BufferType = BufferType;\n```\n\n### A simple example\n\nBelow is an example of a parser for a simple protocol. Each message is a\nUTF-8 string, prefixed with a big-endian unsigned 32-bit integer used as a\nlength specifier.\n\n```javascript\nvar strtok = require('strtok');\nvar Token = require('token-types');\n\nvar s = ... // a net.Stream workalike\n\nvar numBytes = -1;\n\nstrtok.parse(s, function(v, cb) {\n    if (v === undefined) {\n        return Token.UINT32_BE;\n    }\n\n    if (numBytes === -1) {\n        numBytes = v;\n        return new Token.StringType(v, 'utf-8');\n    }\n\n    console.log('Read ' + v);\n    numBytes = -1;\n    return Token.UINT32_BE;\n});\n```\n\nWhen the callback is first invoked, we aren't in the midst of reading a\nmessage, so we ask for a `UINT32_BE` to get the length of the subsequent\nstring. At this point, on every invocation of our callback, we use our\ninternal `numBytes` variable to determine if we're reading a number-of-bytes\nvalue or a string value. In the former case, we're called with the\nnumber-of-bytes value and return a `StringType` instance that knows to read\nthe specified number of bytes from the stream. In the latter case, we get the\nstring itself, log it, and then ask for a length again. Lather, rinse,\nrepeat.\n\n### A more complex example\n\nThe\n[examples/msgpack/msgpack.js](https://github.com/Borewit/strtok2/blob/master/examples/msgpack/msgpack.js)\nfile contains an implementation of the [MsgPack serialization\nspec](http://web.archive.org/web/20100628121422/http://redmine.msgpack.org/projects/msgpack/wiki/FormatSpec).\n\n## Performance\n\nAn example run of the built-in `examples/msgpack/bench.js`:\n\n    json\n      pack:   14674 ms (100% of json)\n      unpack: 50479 ms (100% of json)\n\n    native\n      pack:   15334 ms (104% of json)\n      unpack: 32835 ms (65% of json)\n\n    strtok\n      pack:   15861 ms (108% of json)\n      unpack: 46650 ms (92% of json)\n      \n[npm-url]: https://npmjs.org/package/strtok2\n[npm-image]: https://badge.fury.io/js/strtok2.svg\n[npm-downloads-image]: http://img.shields.io/npm/dm/strtok2.svg\n\n[travis-url]: https://travis-ci.org/Borewit/strtok2\n[travis-image]: https://api.travis-ci.org/Borewit/strtok2.svg?branch=master","_attachments":{},"homepage":"https://github.com/Borewit/strtok2#readme","bugs":{"url":"https://github.com/Borewit/strtok2/issues"},"license":"BSD-3-Clause"}