From: Feross Aboukhadijeh Date: Mon, 3 Feb 2014 08:14:20 +0000 (-0800) Subject: Add utf16le support X-Git-Url: https://zoso.dev/?a=commitdiff_plain;h=c78cb61082bdb8a764647858a41d2334f983f7d8;p=buffer.git Add utf16le support --- c78cb61082bdb8a764647858a41d2334f983f7d8 diff --cc index.js index 13d9255,ecddf51..7b7d7c6 --- a/index.js +++ b/index.js @@@ -237,12 -236,26 +237,17 @@@ function _binaryWrite (buf, string, off } function _base64Write (buf, string, offset, length) { - var bytes, pos - return Buffer._charsWritten = blitBuffer(base64ToBytes(string), buf, offset, length) + var charsWritten = Buffer._charsWritten = + blitBuffer(base64ToBytes(string), buf, offset, length) + return charsWritten } + function _utf16leWrite (buf, string, offset, length) { + var bytes, pos + return Buffer._charsWritten = blitBuffer(utf16leToBytes(string), buf, offset, length) + } + -function _utf16leWrite (buf, string, offset, length) { - var bytes, pos - return Buffer._charsWritten = blitBuffer(utf16leToBytes(string), buf, offset, length) -} - -function _utf16beWrite (buf, string, offset, length) { - var bytes, pos - return Buffer._charsWritten = blitBuffer(utf16beToBytes(string), buf, offset, length) -} - -function BufferWrite (string, offset, length, encoding) { +Buffer.prototype.write = function (string, offset, length, encoding) { // Support both (string, offset, length, encoding) // and the legacy (string, encoding, offset, length) if (isFinite(offset)) { @@@ -272,19 -285,30 +277,20 @@@ switch (encoding) { case 'hex': return _hexWrite(this, string, offset, length) - case 'utf8': case 'utf-8': - case 'ucs2': // TODO: No support for ucs2 or utf16le encodings yet - case 'ucs-2': - case 'utf16le': - case 'utf-16le': return _utf8Write(this, string, offset, length) - case 'ascii': return _asciiWrite(this, string, offset, length) - case 'binary': return _binaryWrite(this, string, offset, length) - case 'base64': return _base64Write(this, string, offset, length) - + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return _utf16leWrite(this, string, offset, length) - - case 'utf16be': - case 'utf-16be': - return _utf16beWrite(this, string, offset, length) - default: throw new Error('Unknown encoding') } @@@ -306,19 -332,30 +312,20 @@@ Buffer.prototype.toString = function (e switch (encoding) { case 'hex': return _hexSlice(self, start, end) - case 'utf8': case 'utf-8': - case 'ucs2': // TODO: No support for ucs2 or utf16le encodings yet - case 'ucs-2': - case 'utf16le': - case 'utf-16le': return _utf8Slice(self, start, end) - case 'ascii': return _asciiSlice(self, start, end) - case 'binary': return _binarySlice(self, start, end) - case 'base64': return _base64Slice(self, start, end) - + case 'ucs2': + case 'ucs-2': + case 'utf16le': + case 'utf-16le': + return _utf16leSlice(self, start, end) - - case 'utf16be': - case 'utf-16be': - return _utf16beSlice(self, start, end) - default: throw new Error('Unknown encoding') } @@@ -412,8 -449,28 +419,16 @@@ function _hexSlice (buf, start, end) return out } - // http://nodejs.org/api/buffer.html#buffer_buf_slice_start_end + function _utf16leSlice (buf, start, end) { + var bytes = buf.slice(start, end) + var res = '' + for (var i = 0; i < bytes.length; i += 2) { + res += String.fromCharCode(bytes[i] + bytes[i+1] * 256) + } + return res + } + -function _utf16beSlice (buf, start, end) { - var bytes = buf.slice(start, end) - var res = '' - for (var i = 0; i < bytes.length; i += 2) { - res += String.fromCharCode(bytes[i] * 256 + bytes[i+1]) - } - return res -} - -// TODO: add test that modifying the new buffer slice will modify memory in the -// original buffer! Use code from: -// http://nodejs.org/api/buffer.html#buffer_buf_slice_start_end -function BufferSlice (start, end) { +Buffer.prototype.slice = function (start, end) { var len = this.length start = clamp(start, len, 0) end = clamp(end, len, len) @@@ -1003,6 -1150,35 +1018,20 @@@ function asciiToBytes (str) return byteArray } + function utf16leToBytes (str) { + var c, hi, lo + var byteArray = [] + for (var i = 0; i < str.length; i++) { + c = str.charCodeAt(i) + hi = c >> 8 + lo = c % 256 + byteArray.push(lo) + byteArray.push(hi) + } + + return byteArray + } + -function utf16beToBytes (str) { - var c, hi, lo - var byteArray = [] - for (var i = 0; i < str.length; i++) { - c = str.charCodeAt(i) - hi = c >> 8 - lo = c % 256 - byteArray.push(hi) - byteArray.push(lo) - } - - return byteArray -} - function base64ToBytes (str) { return base64.toByteArray(str) } @@@ -1038,15 -1218,18 +1067,15 @@@ function verifuint (value, max) assert(Math.floor(value) === value, 'value has a fractional component') } -/* - * A series of checks to make sure we actually have a signed 32-bit number - */ --function verifsint(value, max, min) { - assert(typeof (value) == 'number', 'cannot write a non-number as a number') ++function verifsint (value, max, min) { + assert(typeof value == 'number', 'cannot write a non-number as a number') assert(value <= max, 'value larger than maximum allowed value') assert(value >= min, 'value smaller than minimum allowed value') assert(Math.floor(value) === value, 'value has a fractional component') } --function verifIEEE754(value, max, min) { - assert(typeof (value) == 'number', 'cannot write a non-number as a number') ++function verifIEEE754 (value, max, min) { + assert(typeof value == 'number', 'cannot write a non-number as a number') assert(value <= max, 'value larger than maximum allowed value') assert(value >= min, 'value smaller than minimum allowed value') } diff --cc test/buffer.js index 7e700e4,dfab08c..b8112cd --- a/test/buffer.js +++ b/test/buffer.js @@@ -1,263 -1,279 +1,281 @@@ -var B = require('../index.js').Buffer; -var test = require('tape'); +var B = require('../index.js').Buffer +var test = require('tape') test('utf8 buffer to base64', function (t) { - t.plan(1); - t.equal( - new B("Ձאab", "utf8").toString("base64"), - '1YHXkGFi' - ); - t.end(); -}); + t.equal( + new B("Ձאab", "utf8").toString("base64"), + '1YHXkGFi' + ) + t.end() +}) test('utf8 buffer to hex', function (t) { - t.plan(1); - t.equal( - new B("Ձאab", "utf8").toString("hex"), - 'd581d7906162' - ); - t.end(); -}); + t.equal( + new B("Ձאab", "utf8").toString("hex"), + 'd581d7906162' + ) + t.end() +}) test('utf8 to utf8', function (t) { - t.plan(1); - t.equal( - new B("öäüõÖÄÜÕ", "utf8").toString("utf8"), - 'öäüõÖÄÜÕ' - ); - t.end(); -}); + t.equal( + new B("öäüõÖÄÜÕ", "utf8").toString("utf8"), + 'öäüõÖÄÜÕ' + ) + t.end() +}) -test('utf16be to hexutf16', function (t) { ++test('utf16le to utf16', function (t) { + t.plan(1); + t.equal( - new B("abcd", "utf16be").toString('hex'), - '0061006200630064' ++ new B(new B("abcd", "utf8").toString("utf16le"), "utf16le").toString("utf8"), ++ 'abcd' + ); + t.end(); + }); + + test('utf16le to hex', function (t) { + t.plan(1); + t.equal( + new B("abcd", "utf16le").toString('hex'), + '6100620063006400' + ); + t.end(); + }); + test('ascii buffer to base64', function (t) { - t.plan(1); - t.equal( - new B("123456!@#$%^", "ascii").toString("base64"), - 'MTIzNDU2IUAjJCVe' - ); - t.end(); -}); + t.equal( + new B("123456!@#$%^", "ascii").toString("base64"), + 'MTIzNDU2IUAjJCVe' + ) + t.end() +}) test('ascii buffer to hex', function (t) { - t.plan(1); - t.equal( - new B("123456!@#$%^", "ascii").toString("hex"), - '31323334353621402324255e' - ); - t.end(); -}); + t.equal( + new B("123456!@#$%^", "ascii").toString("hex"), + '31323334353621402324255e' + ) + t.end() +}) test('base64 buffer to utf8', function (t) { - t.plan(1); - t.equal( - new B("1YHXkGFi", "base64").toString("utf8"), - 'Ձאab' - ); - t.end(); -}); + t.equal( + new B("1YHXkGFi", "base64").toString("utf8"), + 'Ձאab' + ) + t.end() +}) test('hex buffer to utf8', function (t) { - t.plan(1); - t.equal( - new B("d581d7906162", "hex").toString("utf8"), - 'Ձאab' - ); - t.end(); -}); + t.equal( + new B("d581d7906162", "hex").toString("utf8"), + 'Ձאab' + ) + t.end() +}) test('base64 buffer to ascii', function (t) { - t.plan(1); - t.equal( - new B("MTIzNDU2IUAjJCVe", "base64").toString("ascii"), - '123456!@#$%^' - ); - t.end(); -}); + t.equal( + new B("MTIzNDU2IUAjJCVe", "base64").toString("ascii"), + '123456!@#$%^' + ) + t.end() +}) test('hex buffer to ascii', function (t) { - t.plan(1); - t.equal( - new B("31323334353621402324255e", "hex").toString("ascii"), - '123456!@#$%^' - ); - t.end(); -}); -/* -test('utf8 to ascii', function (t) { - t.plan(1); - t.equal( - new B("öäüõÖÄÜÕ", "utf8").toString("ascii"), - new Buffer("öäüõÖÄÜÕ", "utf8").toString("ascii") - ); - t.end(); -}); -*/ + t.equal( + new B("31323334353621402324255e", "hex").toString("ascii"), + '123456!@#$%^' + ) + t.end() +}) test('base64 buffer to binary', function (t) { - t.plan(1); - t.equal( - new B("MTIzNDU2IUAjJCVe", "base64").toString("binary"), - '123456!@#$%^' - ); - t.end(); -}); + t.equal( + new B("MTIzNDU2IUAjJCVe", "base64").toString("binary"), + '123456!@#$%^' + ) + t.end() +}) test('hex buffer to binary', function (t) { - t.plan(1); - t.equal( - new B("31323334353621402324255e", "hex").toString("binary"), - '123456!@#$%^' - ); - t.end(); -}); + t.equal( + new B("31323334353621402324255e", "hex").toString("binary"), + '123456!@#$%^' + ) + t.end() +}) test('utf8 to binary', function (t) { - t.plan(1); - t.equal( - new B("öäüõÖÄÜÕ", "utf8").toString("binary"), - "öäüõÖÄÜÕ" - ); - t.end(); -}); + t.equal( + new B("öäüõÖÄÜÕ", "utf8").toString("binary"), + "öäüõÖÄÜÕ" + ) + t.end() +}) test("hex of write{Uint,Int}{8,16,32}{LE,BE}", function (t) { - t.plan(2*(2*2*2+2)); - var hex = [ - "03", "0300", "0003", "03000000", "00000003", - "fd", "fdff", "fffd", "fdffffff", "fffffffd" - ]; - var reads = [ 3, 3, 3, 3, 3, -3, -3, -3, -3, -3 ]; - ["UInt","Int"].forEach(function(x){ - [8,16,32].forEach(function(y){ - var endianesses = (y === 8) ? [""] : ["LE","BE"]; - endianesses.forEach(function(z){ - var v1 = new B(y / 8); - var writefn = "write" + x + y + z; - var val = (x === "Int") ? -3 : 3; - v1[writefn](val, 0); - t.equal( - v1.toString("hex"), - hex.shift() - ); - var readfn = "read" + x + y + z; - t.equal( - v1[readfn](0), - reads.shift() - ); - }); - }); - }); - t.end(); -}); + t.plan(2*(2*2*2+2)) + var hex = [ + "03", "0300", "0003", "03000000", "00000003", + "fd", "fdff", "fffd", "fdffffff", "fffffffd" + ] + var reads = [ 3, 3, 3, 3, 3, -3, -3, -3, -3, -3 ] + var xs = ["UInt","Int"] + var ys = [8,16,32] + for (var i = 0; i < xs.length; i++) { + var x = xs[i] + for (var j = 0; j < ys.length; j++) { + var y = ys[j] + var endianesses = (y === 8) ? [""] : ["LE","BE"] + for (var k = 0; k < endianesses.length; k++) { + var z = endianesses[k] + + var v1 = new B(y / 8) + var writefn = "write" + x + y + z + var val = (x === "Int") ? -3 : 3 + v1[writefn](val, 0) + t.equal( + v1.toString("hex"), + hex.shift() + ) + var readfn = "read" + x + y + z + t.equal( + v1[readfn](0), + reads.shift() + ) + } + } + } + t.end() +}) test("hex of write{Uint,Int}{8,16,32}{LE,BE} with overflow", function (t) { - t.plan(3*(2*2*2+2)); + t.plan(3*(2*2*2+2)) var hex = [ - "", "03", "00", "030000", "000000", - "", "fd", "ff", "fdffff", "ffffff" - ]; + "", "03", "00", "030000", "000000", + "", "fd", "ff", "fdffff", "ffffff" + ] var reads = [ - undefined, 3, 0, 3, 0, - undefined, 253, -256, 16777213, -256 - ]; - ["UInt","Int"].forEach(function(x){ - [8,16,32].forEach(function(y){ - var endianesses = (y === 8) ? [""] : ["LE","BE"]; - endianesses.forEach(function(z){ - var v1 = new B(y / 8 - 1); - var next = new B(4); - next.writeUInt32BE(0, 0); - var writefn = "write" + x + y + z; - var val = (x === "Int") ? -3 : 3; - v1[writefn](val, 0, true); - t.equal( - v1.toString("hex"), - hex.shift() - ); - // check that nothing leaked to next buffer. - t.equal(next.readUInt32BE(0), 0); - // check that no bytes are read from next buffer. - next.writeInt32BE(~0, 0); - var readfn = "read" + x + y + z; - t.equal( - v1[readfn](0, true), - reads.shift() - ); - }); - }); - }); - t.end(); -}); + undefined, 3, 0, 3, 0, + undefined, 253, -256, 16777213, -256 + ] + var xs = ["UInt","Int"] + var ys = [8,16,32] + for (var i = 0; i < xs.length; i++) { + var x = xs[i] + for (var j = 0; j < ys.length; j++) { + var y = ys[j] + var endianesses = (y === 8) ? [""] : ["LE","BE"] + for (var k = 0; k < endianesses.length; k++) { + var z = endianesses[k] + + var v1 = new B(y / 8 - 1) + var next = new B(4) + next.writeUInt32BE(0, 0) + var writefn = "write" + x + y + z + var val = (x === "Int") ? -3 : 3 + v1[writefn](val, 0, true) + t.equal( + v1.toString("hex"), + hex.shift() + ) + // check that nothing leaked to next buffer. + t.equal(next.readUInt32BE(0), 0) + // check that no bytes are read from next buffer. + next.writeInt32BE(~0, 0) + var readfn = "read" + x + y + z + t.equal( + v1[readfn](0, true), + reads.shift() + ) + } + } + } + t.end() +}) test("concat() a varying number of buffers", function (t) { - t.plan(5); - var zero = []; - var one = [ new B('asdf') ]; - var long = []; - for (var i = 0; i < 10; i++) long.push(new B('asdf')); + t.plan(5) + var zero = [] + var one = [ new B('asdf') ] + var long = [] + for (var i = 0; i < 10; i++) long.push(new B('asdf')) - var flatZero = B.concat(zero); - var flatOne = B.concat(one); - var flatLong = B.concat(long); - var flatLongLen = B.concat(long, 40); + var flatZero = B.concat(zero) + var flatOne = B.concat(one) + var flatLong = B.concat(long) + var flatLongLen = B.concat(long, 40) - t.equal(flatZero.length, 0); - t.equal(flatOne.toString(), 'asdf'); - t.equal(flatOne, one[0]); - t.equal(flatLong.toString(), (new Array(10+1).join('asdf'))); - t.equal(flatLongLen.toString(), (new Array(10+1).join('asdf'))); - t.end(); -}); + t.equal(flatZero.length, 0) + t.equal(flatOne.toString(), 'asdf') + t.equal(flatOne, one[0]) + t.equal(flatLong.toString(), (new Array(10+1).join('asdf'))) + t.equal(flatLongLen.toString(), (new Array(10+1).join('asdf'))) + t.end() +}) -test("buffer from buffer", function (t) { - t.plan(1); - var b1 = new B('asdf'); - var b2 = new B(b1); - t.equal(b1.toString('hex'), b2.toString('hex')); - t.end(); -}); +test("new buffer from buffer", function (t) { + var b1 = new B('asdf') + var b2 = new B(b1) + t.equal(b1.toString('hex'), b2.toString('hex')) + t.end() +}) + +test("new buffer from uint8array", function (t) { + if (typeof Uint8Array === 'function') { + var b1 = new Uint8Array([0, 1, 2, 3]) + var b2 = new B(b1) + t.equal(b1[0], 0) + t.equal(b1[1], 1) + t.equal(b1[2], 2) + t.equal(b1[3], 3) + t.equal(b1[4], undefined) + } + t.end() +}) test("fill", function(t) { - t.plan(1); - var b = new B(10); - b.fill(2); - t.equal(b.toString('hex'), '02020202020202020202'); - t.end(); -}); + t.plan(1) + var b = new B(10) + b.fill(2) + t.equal(b.toString('hex'), '02020202020202020202') + t.end() +}) test('copy() empty buffer with sourceEnd=0', function (t) { - t.plan(1); - var source = new B([42]); - var destination = new B([43]); - source.copy(destination, 0, 0, 0); - t.equal(destination.readUInt8(0), 43); - t.end(); -}); + t.plan(1) + var source = new B([42]) + var destination = new B([43]) + source.copy(destination, 0, 0, 0) + t.equal(destination.readUInt8(0), 43) + t.end() +}) test('base64 ignore whitespace', function(t) { - t.plan(1); - var text = "\n YW9ldQ== "; - var buf = new B(text, 'base64'); - t.equal(buf.toString(), 'aoeu'); - t.end(); -}); + t.plan(1) + var text = "\n YW9ldQ== " + var buf = new B(text, 'base64') + t.equal(buf.toString(), 'aoeu') + t.end() +}) test('buffer.slice sets indexes', function (t) { - t.plan(1); - t.equal((new B('hallo')).slice(0, 5).toString(), 'hallo'); - t.end(); -}); + t.plan(1) + t.equal((new B('hallo')).slice(0, 5).toString(), 'hallo') + t.end() +}) test('buffer.slice out of range', function (t) { - t.plan(2); - t.equal((new B('hallo')).slice(0, 10).toString(), 'hallo'); - t.equal((new B('hallo')).slice(10, 2).toString(), ''); - t.end(); -}); + t.plan(2) + t.equal((new B('hallo')).slice(0, 10).toString(), 'hallo') + t.equal((new B('hallo')).slice(10, 2).toString(), '') + t.end() +}) test('base64 strings without padding', function (t) { - t.plan(1); - t.equal((new B('YW9ldQ', 'base64').toString()), 'aoeu'); - t.end(); -}); + t.plan(1) + t.equal((new B('YW9ldQ', 'base64').toString()), 'aoeu') + t.end() +})