diff --git a/esbuild-serve.js b/esbuild-serve.js index 8955c4d9..f9dd3167 100644 --- a/esbuild-serve.js +++ b/esbuild-serve.js @@ -3,21 +3,22 @@ * It attaches the parquet.js exports to a "parquetjs" global variable. * See the example server for how to use it. */ -const {compressionBrowserPlugin, wasmPlugin} = require("./esbuild-plugins"); +const { compressionBrowserPlugin, wasmPlugin } = require("./esbuild-plugins"); // esbuild has TypeScript support by default. It will use .tsconfig -require('esbuild') - .serve({ - servedir: __dirname, - }, { - entryPoints: ['parquet.ts'], - outfile: 'main.js', - define: {"process.env.NODE_DEBUG": "false", "process.env.NODE_ENV": "\"production\"", global: "window" }, - platform: 'browser', - plugins: [compressionBrowserPlugin,wasmPlugin], - sourcemap: "external", - bundle: true, - globalName: 'parquetjs', - inject: ['./esbuild-shims.js'] - }).then(server => { - console.log("serving parquetjs", server) - }) +require('esbuild').context({ + entryPoints: ['parquet.ts'], + outfile: 'main.js', + define: { "process.env.NODE_DEBUG": "false", "process.env.NODE_ENV": "\"production\"", global: "window" }, + platform: 'browser', + plugins: [compressionBrowserPlugin, wasmPlugin], + sourcemap: "external", + bundle: true, + globalName: 'parquetjs', + inject: ['./esbuild-shims.js'] +}).then(context => { + context.serve({ + servedir: __dirname, + }).then(server => { + console.log("serving parquetjs", server) + }) +}) diff --git a/examples/server/README.md b/examples/server/README.md index 0ddee9c6..54cadde0 100644 --- a/examples/server/README.md +++ b/examples/server/README.md @@ -1,8 +1,8 @@ # Example Server This is a toy server that illustrates how to use the parquetjs library built with esbuild. -To run it: +To run it: 1. npm install -1. View and edit the files in `views` to taste +1. View and edit the files in `views` to taste 1. node app.js -1. Build and serve the parquetjs bundle in the main parquetjs directory: `npm run serve` +1. Build and serve the parquetjs bundle in the main parquetjs directory: `npm run serve` 1. visit `http://localhost:3000` and click buttons, do things in the console. diff --git a/lib/bufferReader.ts b/lib/bufferReader.ts index 9c274dd8..bf519dea 100644 --- a/lib/bufferReader.ts +++ b/lib/bufferReader.ts @@ -72,7 +72,7 @@ export default class BufferReader { const buffer = await this.envelopeReader.readFn(start, finish - start); processQueue.forEach(async d => { - d.resolve(buffer.slice(d.offset - start, d.offset + d.length - start)); + d.resolve(buffer.subarray(d.offset - start, d.offset + d.length - start)); }); }; diff --git a/lib/codec/plain.ts b/lib/codec/plain.ts index 22871e08..a5fbb359 100644 --- a/lib/codec/plain.ts +++ b/lib/codec/plain.ts @@ -205,7 +205,6 @@ function decodeValues_DOUBLE(cursor: Cursor, count: number) { return values; } -// Waylands reminder to check again function encodeValues_BYTE_ARRAY(values: Array) { let buf_len = 0; const returnedValues: Array = []; @@ -231,7 +230,7 @@ function decodeValues_BYTE_ARRAY(cursor: Cursor, count: number) { for (let i = 0; i < count; ++i) { let len = cursor.buffer.readUInt32LE(cursor.offset); cursor.offset += 4; - values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len)); + values.push(cursor.buffer.subarray(cursor.offset, cursor.offset + len)); cursor.offset += len; } @@ -272,7 +271,7 @@ function decodeValues_FIXED_LEN_BYTE_ARRAY( for (let i = 0; i < count; ++i) { values.push( - cursor.buffer.slice(cursor.offset, cursor.offset + typeLength) + cursor.buffer.subarray(cursor.offset, cursor.offset + typeLength) ); cursor.offset += typeLength; } diff --git a/lib/codec/plain_dictionary.ts b/lib/codec/plain_dictionary.ts index 70121462..963baec8 100644 --- a/lib/codec/plain_dictionary.ts +++ b/lib/codec/plain_dictionary.ts @@ -2,7 +2,7 @@ import * as rle from './rle' import { Cursor, Options } from './types' export const decodeValues = function(type: string, cursor: Cursor, count: number, opts: Options) { - const bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset+1).readInt8(0); + const bitWidth = cursor.buffer.subarray(cursor.offset, cursor.offset+1).readInt8(0); cursor.offset += 1; return rle.decodeValues(type, cursor, count, Object.assign({}, opts, { disableEnvelope: true, bitWidth })); }; diff --git a/lib/reader.ts b/lib/reader.ts index e61e55d4..113fb517 100644 --- a/lib/reader.ts +++ b/lib/reader.ts @@ -441,7 +441,7 @@ export class ParquetEnvelopeReader { return Promise.reject('external references are not supported'); } - return Promise.resolve(buffer.slice(offset, offset + length)); + return Promise.resolve(buffer.subarray(offset, offset + length)); }; let closeFn = () => ({}); @@ -503,7 +503,7 @@ export class ParquetEnvelopeReader { if (blob.arrayBuffer !== undefined) { const arrayBuffer = await blob.arrayBuffer(); const uint8Array: Uint8Array = new Uint8Array(arrayBuffer); - return new Buffer(uint8Array); + return Buffer.from(uint8Array); } //Assumed to be a Readable like object @@ -834,7 +834,7 @@ async function decodePage(cursor: Cursor, opts: Options): Promise { const pageHeader = new NewPageHeader(); const headerOffset = cursor.offset; - const headerSize = parquet_util.decodeThrift(pageHeader, cursor.buffer.slice(cursor.offset)); + const headerSize = parquet_util.decodeThrift(pageHeader, cursor.buffer.subarray(cursor.offset)); cursor.offset += headerSize; const pageType = parquet_util.getThriftEnum( @@ -926,7 +926,7 @@ async function decodeDictionaryPage(cursor: Cursor, header: parquet_thrift.PageH let dictCursor = { offset: 0, - buffer: cursor.buffer.slice(cursor.offset,cursorEnd), + buffer: cursor.buffer.subarray(cursor.offset,cursorEnd), size: cursorEnd - cursor.offset }; @@ -935,7 +935,7 @@ async function decodeDictionaryPage(cursor: Cursor, header: parquet_thrift.PageH if (opts.compression && opts.compression !== 'UNCOMPRESSED') { let valuesBuf = await parquet_compression.inflate( opts.compression, - dictCursor.buffer.slice(dictCursor.offset,cursorEnd)); + dictCursor.buffer.subarray(dictCursor.offset,cursorEnd)); dictCursor = { buffer: valuesBuf, @@ -963,7 +963,7 @@ async function decodeDataPage(cursor: Cursor, header: parquet_thrift.PageHeader, if (opts.compression && opts.compression !== 'UNCOMPRESSED') { let valuesBuf = await parquet_compression.inflate( opts.compression, - cursor.buffer.slice(cursor.offset, cursorEnd)); + cursor.buffer.subarray(cursor.offset, cursorEnd)); valuesBufCursor = { buffer: valuesBuf, @@ -1090,7 +1090,7 @@ async function decodeDataPageV2(cursor: Cursor, header: parquet_thrift.PageHeade if (dataPageHeaderV2.is_compressed) { let valuesBuf = await parquet_compression.inflate( opts.compression!, - cursor.buffer.slice(cursor.offset, cursorEnd)); + cursor.buffer.subarray(cursor.offset, cursorEnd)); valuesBufCursor = { buffer: valuesBuf, diff --git a/test/bloomFilterIntegration.ts b/test/bloomFilterIntegration.ts index aa41de17..178c245b 100644 --- a/test/bloomFilterIntegration.ts +++ b/test/bloomFilterIntegration.ts @@ -32,7 +32,7 @@ const sampleColumnHeaders = async (filename: string) => { while (cursor.offset < cursor.size) { const pageHeader = new parquet_thrift.PageHeader(); - cursor.offset += decodeThrift(pageHeader, cursor.buffer.slice(cursor.offset)); + cursor.offset += decodeThrift(pageHeader, cursor.buffer.subarray(cursor.offset)); pages.push(pageHeader); cursor.offset += pageHeader.compressed_page_size; } diff --git a/test/integration.js b/test/integration.js index dd181386..6832d69d 100644 --- a/test/integration.js +++ b/test/integration.js @@ -152,7 +152,7 @@ async function sampleColumnHeaders() { while (cursor.offset < cursor.size) { const pageHeader = new parquet_thrift.PageHeader(); - cursor.offset += parquet_util.decodeThrift(pageHeader, cursor.buffer.slice(cursor.offset)); + cursor.offset += parquet_util.decodeThrift(pageHeader, cursor.buffer.subarray(cursor.offset)); pages.push(pageHeader); cursor.offset += pageHeader.compressed_page_size; } diff --git a/test/lib/bufferReader.test.js b/test/lib/bufferReader.test.js index a01594cd..5682f09f 100644 --- a/test/lib/bufferReader.test.js +++ b/test/lib/bufferReader.test.js @@ -1,4 +1,4 @@ -import chai, {expect} from "chai" +import chai, { expect } from "chai" import sinon from "sinon" import sinonChai from "sinon-chai"; import sinonChaiInOrder from 'sinon-chai-in-order'; @@ -30,7 +30,7 @@ describe("bufferReader", () => { it("only enqueues an item and reads on flushing the queue", async () => { const mockResolve = sinon.spy(); const mockResolve2 = sinon.spy(); - reader.envelopeReader = {readFn: sinon.fake.returns("buffer")} + reader.envelopeReader = { readFn: sinon.fake.returns(Buffer.from("buffer", "utf8")) } reader.queue = [{ offset: 1, @@ -44,15 +44,16 @@ describe("bufferReader", () => { await reader.processQueue(); - sinon.assert.calledWith(mockResolve, "b") - sinon.assert.calledWith(mockResolve2, "uffe") + + sinon.assert.calledWith(mockResolve, Buffer.from("b", "utf8")); + sinon.assert.calledWith(mockResolve2, Buffer.from("uffe", "utf8")); }) it("enqueues items and then reads them", async () => { const mockResolve = sinon.spy(); const mockResolve2 = sinon.spy(); reader.maxLength = 1; - reader.envelopeReader = {readFn: sinon.fake.returns("buffer")} + reader.envelopeReader = { readFn: sinon.fake.returns(Buffer.from("buffer", "utf8")) } reader.queue = [{ offset: 1, @@ -66,81 +67,81 @@ describe("bufferReader", () => { await reader.processQueue(); - sinon.assert.calledWith(mockResolve, "b") - sinon.assert.calledWith(mockResolve2, "uffe") + sinon.assert.calledWith(mockResolve, Buffer.from("b", "utf8")); + sinon.assert.calledWith(mockResolve2, Buffer.from("uffe", "utf8")); }) it("enqueues items and reads them in order", async () => { const mockResolve = sinon.spy(); - reader.envelopeReader = {readFn: sinon.fake.returns("thisisalargebuffer")} + reader.envelopeReader = { readFn: sinon.fake.returns(Buffer.from("thisisalargebuffer", "utf8")) } reader.queue = [{ - offset: 1, - length: 4, - resolve: mockResolve, - }, { - offset: 5, - length: 2, - resolve: mockResolve, - }, { - offset: 7, - length: 1, - resolve: mockResolve, - }, { - offset: 8, - length: 5, - resolve: mockResolve, - }, { - offset: 13, - length: 6, - resolve: mockResolve, - } + offset: 1, + length: 4, + resolve: mockResolve, + }, { + offset: 5, + length: 2, + resolve: mockResolve, + }, { + offset: 7, + length: 1, + resolve: mockResolve, + }, { + offset: 8, + length: 5, + resolve: mockResolve, + }, { + offset: 13, + length: 6, + resolve: mockResolve, + } ]; await reader.processQueue(); - expect(mockResolve).inOrder.to.have.been.calledWith("this") - .subsequently.calledWith("is") - .subsequently.calledWith("a") - .subsequently.calledWith("large") - .subsequently.calledWith("buffer"); + expect(mockResolve).inOrder.to.have.been.calledWith(Buffer.from("this", "utf8")) + .subsequently.calledWith(Buffer.from("is", "utf8")) + .subsequently.calledWith(Buffer.from("a", "utf8")) + .subsequently.calledWith(Buffer.from("large", "utf8")) + .subsequently.calledWith(Buffer.from("buffer", "utf8")); }) it("should read even if the maxSpan has been exceeded", async () => { const mockResolve = sinon.spy(); reader.maxSpan = 5; - reader.envelopeReader = {readFn: sinon.fake.returns("willslicefrombeginning")} + reader.envelopeReader = { readFn: sinon.fake.returns(Buffer.from("willslicefrombeginning", "utf8")) } reader.queue = [{ - offset: 1, - length: 4, - resolve: mockResolve, - }, { - offset: 10, - length: 4, - resolve: mockResolve, - }, { - offset: 10, - length: 9, - resolve: mockResolve, - }, { - offset: 10, - length: 13, - resolve: mockResolve, - }, { - offset: 10, - length: 22, - resolve: mockResolve, - } + offset: 1, + length: 4, + resolve: mockResolve, + }, { + offset: 10, + length: 4, + resolve: mockResolve, + }, { + offset: 10, + length: 9, + resolve: mockResolve, + }, { + offset: 10, + length: 13, + resolve: mockResolve, + }, { + offset: 10, + length: 22, + resolve: mockResolve, + } ]; await reader.processQueue(); - expect(mockResolve).inOrder.to.have.been.calledWith("will") - .subsequently.calledWith("will") - .subsequently.calledWith("willslice") - .subsequently.calledWith("willslicefrom") - .subsequently.calledWith("willslicefrombeginning"); + expect(mockResolve).inOrder.to.have.been.calledWith(Buffer.from("will", "utf8")) + .subsequently.calledWith(Buffer.from("will", "utf8")) + .subsequently.calledWith(Buffer.from("willslice", "utf8")) + .subsequently.calledWith(Buffer.from("willslicefrom", "utf8")) + .subsequently.calledWith(Buffer.from("willslicefrombeginning", "utf8")); }) }) })