Update Deprecated Function Calls (#118)

- Buffer.slice -> Buffer.subarray (and correct test that wasn't using
buffers)
- new Buffer(array) -> Buffer.from(array)
- Fix issue with `npm run serve`

Via looking into #117 As `subarray` is slightly faster in the browser
shim.

esbuild-serve.js

@@ -3,21 +3,22 @@
  * It attaches the parquet.js exports to a "parquetjs" global variable.
  * See the example server for how to use it.
  */
-const {compressionBrowserPlugin, wasmPlugin} = require("./esbuild-plugins");
+const { compressionBrowserPlugin, wasmPlugin } = require("./esbuild-plugins");
 // esbuild has TypeScript support by default. It will use .tsconfig
-require('esbuild')
-      .serve({
-        servedir: __dirname,
-      }, {
-        entryPoints: ['parquet.ts'],
-        outfile: 'main.js',
-        define: {"process.env.NODE_DEBUG": "false", "process.env.NODE_ENV": "\"production\"", global: "window" },
-        platform: 'browser',
-        plugins: [compressionBrowserPlugin,wasmPlugin],
-        sourcemap: "external",
-        bundle: true,
-        globalName: 'parquetjs',
-        inject: ['./esbuild-shims.js']
-      }).then(server => {
-          console.log("serving parquetjs", server)
-      })
+require('esbuild').context({
+  entryPoints: ['parquet.ts'],
+  outfile: 'main.js',
+  define: { "process.env.NODE_DEBUG": "false", "process.env.NODE_ENV": "\"production\"", global: "window" },
+  platform: 'browser',
+  plugins: [compressionBrowserPlugin, wasmPlugin],
+  sourcemap: "external",
+  bundle: true,
+  globalName: 'parquetjs',
+  inject: ['./esbuild-shims.js']
+}).then(context => {
+  context.serve({
+    servedir: __dirname,
+  }).then(server => {
+    console.log("serving parquetjs", server)
+  })
+})

examples/server/README.md

@@ -1,8 +1,8 @@
 # Example Server
 This is a toy server that illustrates how to use the parquetjs library built with esbuild.
-To run it: 
+To run it:
 1. npm install
-1. View and edit the files in `views` to taste   
+1. View and edit the files in `views` to taste
 1. node app.js
-1. Build and serve the parquetjs bundle in the main parquetjs directory: `npm run serve`    
+1. Build and serve the parquetjs bundle in the main parquetjs directory: `npm run serve`
 1. visit `http://localhost:3000` and click buttons, do things in the console.

lib/bufferReader.ts

@@ -72,7 +72,7 @@ export default class BufferReader {
       const buffer = await this.envelopeReader.readFn(start, finish - start);
 
       processQueue.forEach(async d => {
-        d.resolve(buffer.slice(d.offset - start, d.offset + d.length - start));
+        d.resolve(buffer.subarray(d.offset - start, d.offset + d.length - start));
       });
     };
 

lib/codec/plain.ts

@@ -205,7 +205,6 @@ function decodeValues_DOUBLE(cursor: Cursor, count: number) {
   return values;
 }
 
-// Waylands reminder to check again
 function encodeValues_BYTE_ARRAY(values: Array<Uint8Array>) {
   let buf_len = 0;
   const returnedValues: Array<Buffer> = [];
@@ -231,7 +230,7 @@ function decodeValues_BYTE_ARRAY(cursor: Cursor, count: number) {
   for (let i = 0; i < count; ++i) {
     let len = cursor.buffer.readUInt32LE(cursor.offset);
     cursor.offset += 4;
-    values.push(cursor.buffer.slice(cursor.offset, cursor.offset + len));
+    values.push(cursor.buffer.subarray(cursor.offset, cursor.offset + len));
     cursor.offset += len;
   }
 
@@ -272,7 +271,7 @@ function decodeValues_FIXED_LEN_BYTE_ARRAY(
 
   for (let i = 0; i < count; ++i) {
     values.push(
-      cursor.buffer.slice(cursor.offset, cursor.offset + typeLength)
+      cursor.buffer.subarray(cursor.offset, cursor.offset + typeLength)
     );
     cursor.offset += typeLength;
   }

lib/codec/plain_dictionary.ts

@@ -2,7 +2,7 @@ import * as rle from './rle'
 import { Cursor, Options } from './types'
 
 export const decodeValues = function(type: string, cursor: Cursor, count: number, opts: Options) {
-  const bitWidth = cursor.buffer.slice(cursor.offset, cursor.offset+1).readInt8(0);
+  const bitWidth = cursor.buffer.subarray(cursor.offset, cursor.offset+1).readInt8(0);
   cursor.offset += 1;
   return rle.decodeValues(type, cursor, count, Object.assign({}, opts, { disableEnvelope: true, bitWidth }));
 };

lib/reader.ts

@@ -441,7 +441,7 @@ export class ParquetEnvelopeReader {
         return Promise.reject('external references are not supported');
       }
 
-      return Promise.resolve(buffer.slice(offset, offset + length));
+      return Promise.resolve(buffer.subarray(offset, offset + length));
     };
 
     let closeFn = () => ({});
@@ -503,7 +503,7 @@ export class ParquetEnvelopeReader {
     if (blob.arrayBuffer !== undefined) {
       const arrayBuffer = await blob.arrayBuffer();
       const uint8Array: Uint8Array = new Uint8Array(arrayBuffer);
-      return new Buffer(uint8Array);
+      return Buffer.from(uint8Array);
     }
 
     //Assumed to be a Readable like object
@@ -834,7 +834,7 @@ async function decodePage(cursor: Cursor, opts: Options): Promise<PageData> {
   const pageHeader = new NewPageHeader();
 
   const headerOffset = cursor.offset;
-  const headerSize = parquet_util.decodeThrift(pageHeader, cursor.buffer.slice(cursor.offset));
+  const headerSize = parquet_util.decodeThrift(pageHeader, cursor.buffer.subarray(cursor.offset));
   cursor.offset += headerSize;
 
   const pageType = parquet_util.getThriftEnum(
@@ -926,7 +926,7 @@ async function decodeDictionaryPage(cursor: Cursor, header: parquet_thrift.PageH
 
   let dictCursor = {
     offset: 0,
-    buffer: cursor.buffer.slice(cursor.offset,cursorEnd),
+    buffer: cursor.buffer.subarray(cursor.offset,cursorEnd),
     size: cursorEnd - cursor.offset
   };
 
@@ -935,7 +935,7 @@ async function decodeDictionaryPage(cursor: Cursor, header: parquet_thrift.PageH
   if (opts.compression && opts.compression !== 'UNCOMPRESSED') {
     let valuesBuf = await parquet_compression.inflate(
         opts.compression,
-        dictCursor.buffer.slice(dictCursor.offset,cursorEnd));
+        dictCursor.buffer.subarray(dictCursor.offset,cursorEnd));
 
     dictCursor = {
       buffer: valuesBuf,
@@ -963,7 +963,7 @@ async function decodeDataPage(cursor: Cursor, header: parquet_thrift.PageHeader,
   if (opts.compression && opts.compression !== 'UNCOMPRESSED') {
     let valuesBuf = await parquet_compression.inflate(
         opts.compression,
-        cursor.buffer.slice(cursor.offset, cursorEnd));
+        cursor.buffer.subarray(cursor.offset, cursorEnd));
 
     valuesBufCursor = {
       buffer: valuesBuf,
@@ -1090,7 +1090,7 @@ async function decodeDataPageV2(cursor: Cursor, header: parquet_thrift.PageHeade
   if (dataPageHeaderV2.is_compressed) {
     let valuesBuf = await parquet_compression.inflate(
         opts.compression!,
-        cursor.buffer.slice(cursor.offset, cursorEnd));
+        cursor.buffer.subarray(cursor.offset, cursorEnd));
 
     valuesBufCursor = {
       buffer: valuesBuf,

test/bloomFilterIntegration.ts

@@ -32,7 +32,7 @@ const sampleColumnHeaders = async (filename: string) => {
 
   while (cursor.offset < cursor.size) {
     const pageHeader = new parquet_thrift.PageHeader();
-    cursor.offset += decodeThrift(pageHeader, cursor.buffer.slice(cursor.offset));
+    cursor.offset += decodeThrift(pageHeader, cursor.buffer.subarray(cursor.offset));
     pages.push(pageHeader);
     cursor.offset += pageHeader.compressed_page_size;
   }

test/integration.js

@@ -152,7 +152,7 @@ async function sampleColumnHeaders() {
 
   while (cursor.offset < cursor.size) {
     const pageHeader = new parquet_thrift.PageHeader();
-    cursor.offset += parquet_util.decodeThrift(pageHeader, cursor.buffer.slice(cursor.offset));
+    cursor.offset += parquet_util.decodeThrift(pageHeader, cursor.buffer.subarray(cursor.offset));
     pages.push(pageHeader);
     cursor.offset += pageHeader.compressed_page_size;
   }

test/lib/bufferReader.test.js

@@ -1,4 +1,4 @@
-import chai, {expect} from "chai"
+import chai, { expect } from "chai"
 import sinon from "sinon"
 import sinonChai from "sinon-chai";
 import sinonChaiInOrder from 'sinon-chai-in-order';
@@ -30,7 +30,7 @@ describe("bufferReader", () => {
     it("only enqueues an item and reads on flushing the queue", async () => {
       const mockResolve = sinon.spy();
       const mockResolve2 = sinon.spy();
-      reader.envelopeReader = {readFn: sinon.fake.returns("buffer")}
+      reader.envelopeReader = { readFn: sinon.fake.returns(Buffer.from("buffer", "utf8")) }
 
       reader.queue = [{
         offset: 1,
@@ -44,15 +44,16 @@ describe("bufferReader", () => {
 
       await reader.processQueue();
 
-      sinon.assert.calledWith(mockResolve, "b")
-      sinon.assert.calledWith(mockResolve2, "uffe")
+
+      sinon.assert.calledWith(mockResolve, Buffer.from("b", "utf8"));
+      sinon.assert.calledWith(mockResolve2, Buffer.from("uffe", "utf8"));
     })
 
     it("enqueues items and then reads them", async () => {
       const mockResolve = sinon.spy();
       const mockResolve2 = sinon.spy();
       reader.maxLength = 1;
-      reader.envelopeReader = {readFn: sinon.fake.returns("buffer")}
+      reader.envelopeReader = { readFn: sinon.fake.returns(Buffer.from("buffer", "utf8")) }
 
       reader.queue = [{
         offset: 1,
@@ -66,81 +67,81 @@ describe("bufferReader", () => {
 
       await reader.processQueue();
 
-      sinon.assert.calledWith(mockResolve, "b")
-      sinon.assert.calledWith(mockResolve2, "uffe")
+      sinon.assert.calledWith(mockResolve, Buffer.from("b", "utf8"));
+      sinon.assert.calledWith(mockResolve2, Buffer.from("uffe", "utf8"));
     })
 
     it("enqueues items and reads them in order", async () => {
       const mockResolve = sinon.spy();
-      reader.envelopeReader = {readFn: sinon.fake.returns("thisisalargebuffer")}
+      reader.envelopeReader = { readFn: sinon.fake.returns(Buffer.from("thisisalargebuffer", "utf8")) }
 
       reader.queue = [{
-          offset: 1,
-          length: 4,
-          resolve: mockResolve,
-        }, {
-          offset: 5,
-          length: 2,
-          resolve: mockResolve,
-        }, {
-          offset: 7,
-          length: 1,
-          resolve: mockResolve,
-        }, {
-          offset: 8,
-          length: 5,
-          resolve: mockResolve,
-        }, {
-          offset: 13,
-          length: 6,
-          resolve: mockResolve,
-        }
+        offset: 1,
+        length: 4,
+        resolve: mockResolve,
+      }, {
+        offset: 5,
+        length: 2,
+        resolve: mockResolve,
+      }, {
+        offset: 7,
+        length: 1,
+        resolve: mockResolve,
+      }, {
+        offset: 8,
+        length: 5,
+        resolve: mockResolve,
+      }, {
+        offset: 13,
+        length: 6,
+        resolve: mockResolve,
+      }
       ];
 
       await reader.processQueue();
 
-      expect(mockResolve).inOrder.to.have.been.calledWith("this")
-        .subsequently.calledWith("is")
-        .subsequently.calledWith("a")
-        .subsequently.calledWith("large")
-        .subsequently.calledWith("buffer");
+      expect(mockResolve).inOrder.to.have.been.calledWith(Buffer.from("this", "utf8"))
+        .subsequently.calledWith(Buffer.from("is", "utf8"))
+        .subsequently.calledWith(Buffer.from("a", "utf8"))
+        .subsequently.calledWith(Buffer.from("large", "utf8"))
+        .subsequently.calledWith(Buffer.from("buffer", "utf8"));
     })
 
     it("should read even if the maxSpan has been exceeded", async () => {
       const mockResolve = sinon.spy();
       reader.maxSpan = 5;
-      reader.envelopeReader = {readFn: sinon.fake.returns("willslicefrombeginning")}
+      reader.envelopeReader = { readFn: sinon.fake.returns(Buffer.from("willslicefrombeginning", "utf8")) }
 
       reader.queue = [{
-          offset: 1,
-          length: 4,
-          resolve: mockResolve,
-        }, {
-          offset: 10,
-          length: 4,
-          resolve: mockResolve,
-        }, {
-          offset: 10,
-          length: 9,
-          resolve: mockResolve,
-        }, {
-          offset: 10,
-          length: 13,
-          resolve: mockResolve,
-        }, {
-          offset: 10,
-          length: 22,
-          resolve: mockResolve,
-        }
+        offset: 1,
+        length: 4,
+        resolve: mockResolve,
+      }, {
+        offset: 10,
+        length: 4,
+        resolve: mockResolve,
+      }, {
+        offset: 10,
+        length: 9,
+        resolve: mockResolve,
+      }, {
+        offset: 10,
+        length: 13,
+        resolve: mockResolve,
+      }, {
+        offset: 10,
+        length: 22,
+        resolve: mockResolve,
+      }
       ];
 
       await reader.processQueue();
 
-      expect(mockResolve).inOrder.to.have.been.calledWith("will")
-        .subsequently.calledWith("will")
-        .subsequently.calledWith("willslice")
-        .subsequently.calledWith("willslicefrom")
-        .subsequently.calledWith("willslicefrombeginning");
+      expect(mockResolve).inOrder.to.have.been.calledWith(Buffer.from("will", "utf8"))
+        .subsequently.calledWith(Buffer.from("will", "utf8"))
+        .subsequently.calledWith(Buffer.from("willslice", "utf8"))
+        .subsequently.calledWith(Buffer.from("willslicefrom", "utf8"))
+        .subsequently.calledWith(Buffer.from("willslicefrombeginning", "utf8"));
     })
   })
 })