Merge pull request #156 from msgpack/decode_multi

gfx · web-flow · commit 57b5646f5b11 · 2021-02-15T22:53:54.000+09:00
introduce decodeMulti()
diff --git a/README.md b/README.md
@@ -47,9 +47,10 @@ deepStrictEqual(decode(encoded), object);
     - [`EncodeOptions`](#encodeoptions)
   - [`decode(buffer: ArrayLike<number> | BufferSource, options?: DecodeOptions): unknown`](#decodebuffer-arraylikenumber--buffersource-options-decodeoptions-unknown)
     - [`DecodeOptions`](#decodeoptions)
+  - [`decodeMulti(buffer: ArrayLike<number> | BufferSource, options?: DecodeOptions): Generator<unknown, void, unknown>`](#decodemultibuffer-arraylikenumber--buffersource-options-decodeoptions-generatorunknown-void-unknown)
   - [`decodeAsync(stream: ReadableStreamLike<ArrayLike<number> | BufferSource>, options?: DecodeAsyncOptions): Promise<unknown>`](#decodeasyncstream-readablestreamlikearraylikenumber--buffersource-options-decodeasyncoptions-promiseunknown)
   - [`decodeArrayStream(stream: ReadableStreamLike<ArrayLike<number> | BufferSource>, options?: DecodeAsyncOptions): AsyncIterable<unknown>`](#decodearraystreamstream-readablestreamlikearraylikenumber--buffersource-options-decodeasyncoptions-asynciterableunknown)
-  - [`decodeStream(stream: ReadableStreamLike<ArrayLike<number> | BufferSource>, options?: DecodeAsyncOptions): AsyncIterable<unknown>`](#decodestreamstream-readablestreamlikearraylikenumber--buffersource-options-decodeasyncoptions-asynciterableunknown)
+  - [`decodeMultiStream(stream: ReadableStreamLike<ArrayLike<number> | BufferSource>, options?: DecodeAsyncOptions): AsyncIterable<unknown>`](#decodemultistreamstream-readablestreamlikearraylikenumber--buffersource-options-decodeasyncoptions-asynciterableunknown)
   - [Reusing Encoder and Decoder instances](#reusing-encoder-and-decoder-instances)
 - [Extension Types](#extension-types)
     - [ExtensionCodec context](#extensioncodec-context)
@@ -87,7 +88,7 @@ npm install @msgpack/msgpack
 
 ### `encode(data: unknown, options?: EncodeOptions): Uint8Array`
 
-It encodes `data` and returns a byte array as `Uint8Array`, throwing errors if `data` is, or includes, a non-serializable object such as a `function` or a `symbol`.
+It encodes `data` into a single MessagePack-encoded object, and returns a byte array as `Uint8Array`, throwing errors if `data` is, or includes, a non-serializable object such as a `function` or a `symbol`.
 
 for example:
 
@@ -98,7 +99,7 @@ const encoded: Uint8Array = encode({ foo: "bar" });
 console.log(encoded);
 ```
 
-If you'd like to convert the uint8array to a NodeJS `Buffer`, use `Buffer.from(arrayBuffer, offset, length)` in order not to copy the underlying `ArrayBuffer`, while `Buffer.from(uint8array)` copies it:
+If you'd like to convert an `uint8array` to a NodeJS `Buffer`, use `Buffer.from(arrayBuffer, offset, length)` in order not to copy the underlying `ArrayBuffer`, while `Buffer.from(uint8array)` copies it:
 
 ```typescript
 import { encode } from "@msgpack/msgpack";
@@ -114,7 +115,7 @@ console.log(buffer);
 
 Name|Type|Default
 ----|----|----
-extensionCodec | ExtensionCodec | `ExtensinCodec.defaultCodec`
+extensionCodec | ExtensionCodec | `ExtensionCodec.defaultCodec`
 maxDepth | number | `100`
 initialBufferSize | number | `2048`
 sortKeys | boolean | false
@@ -125,10 +126,12 @@ context | user-defined | -
 
 ### `decode(buffer: ArrayLike<number> | BufferSource, options?: DecodeOptions): unknown`
 
-It decodes `buffer` encoded in MessagePack, and returns a decoded object as `unknown`.
+It decodes `buffer` that includes a MessagePack-encoded object, and returns the decoded object typed `unknown`.
 
 `buffer` must be an array of bytes, which is typically `Uint8Array` or `ArrayBuffer`. `BufferSource` is defined as `ArrayBuffer | ArrayBufferView`.
 
+In addition, `buffer` can include a single encoded object. If the `buffer` includes extra bytes after an object, it will throw `RangeError`. To decode `buffer` that includes multiple encoded objects, use `decodeMulti()` or `decodeMultiStream()` (recommended) instead.
+
 for example:
 
 ```typescript
@@ -155,9 +158,27 @@ context | user-defined | -
 
 You can use `max${Type}Length` to limit the length of each type decoded.
 
+### `decodeMulti(buffer: ArrayLike<number> | BufferSource, options?: DecodeOptions): Generator<unknown, void, unknown>`
+
+It decodes `buffer` that includes multiple MessagePack-encoded objects, and returns decoded objects as a generator. That is, this is a synchronous variant for `decodeMultiStream()`.
+
+This function is not recommended to decode a MessagePack binary via I/O stream including sockets because it's synchronous. Instead, `decodeMultiStream()` decodes it asynchronously, typically spending less time and memory.
+
+for example:
+
+```typescript
+import { decode } from "@msgpack/msgpack";
+
+const encoded: Uint8Array;
+
+for (const object of decodeMulti(encoded)) {
+  console.log(object);
+}
+```
+
 ### `decodeAsync(stream: ReadableStreamLike<ArrayLike<number> | BufferSource>, options?: DecodeAsyncOptions): Promise<unknown>`
 
-It decodes `stream`, where `ReadableStreamLike<T>` is defined as `ReadableStream<T> | AsyncIterable<T>`, in an async iterable of byte arrays, and returns decoded object as `unknown` type, wrapped in `Promise`. This function works asynchronously.
+It decodes `stream`, where `ReadableStreamLike<T>` is defined as `ReadableStream<T> | AsyncIterable<T>`, in an async iterable of byte arrays, and returns decoded object as `unknown` type, wrapped in `Promise`. This function works asynchronously. This is an async variant for `decode()`.
 
 `DecodeAsyncOptions` is the same as `DecodeOptions` for `decode()`.
 
@@ -178,9 +199,7 @@ if (contentType && contentType.startsWith(MSGPACK_TYPE) && response.body != null
 
 ### `decodeArrayStream(stream: ReadableStreamLike<ArrayLike<number> | BufferSource>, options?: DecodeAsyncOptions): AsyncIterable<unknown>`
 
-It is alike to `decodeAsync()`, but only accepts an array of items as the input `stream`, and emits the decoded item one by one.
-
-It throws errors when the input is not an array-family.
+It is alike to `decodeAsync()`, but only accepts a `stream` that includes an array of items, and emits a decoded item one by one.
 
 for example:
 
@@ -195,12 +214,11 @@ for await (const item of decodeArrayStream(stream)) {
 }
 ```
 
+### `decodeMultiStream(stream: ReadableStreamLike<ArrayLike<number> | BufferSource>, options?: DecodeAsyncOptions): AsyncIterable<unknown>`
 
-### `decodeStream(stream: ReadableStreamLike<ArrayLike<number> | BufferSource>, options?: DecodeAsyncOptions): AsyncIterable<unknown>`
-
-It is alike to `decodeAsync()` and `decodeArrayStream()`, but the input `stream` consists of independent MessagePack items.
+It is alike to `decodeAsync()` and `decodeArrayStream()`, but the input `stream` must consist of multiple MessagePack-encoded items. This is an asynchronous variant for `decodeMulti()`.
 
-In other words, it decodes an unlimited stream and emits an item one by one.
+In other words, it could decode an unlimited stream and emits a decoded item one by one.
 
 for example:
 
@@ -215,21 +233,7 @@ for await (const item of decodeStream(stream)) {
 }
 ```
 
-If you have a multi-values MessagePack binary, you can use `decodeStream()`, but you need to convert it to a stream or an async generator like this:
-
-```typescript
-// A function that generates an AsyncGenerator
-const createStream = async function* (): AsyncGenerator<Uint8Array> {
-  yield encoded;
-};
-
-const result: Array<unknown> = [];
-
-// Decodes it with for-await
-for await (const item of decodeStream(createStream())) {
-  result.push(item);
-}
-```
+This function is available since v2.4.0; previously it was called as `decodeStream()`.
 
 ### Reusing Encoder and Decoder instances
 
diff --git a/src/Decoder.ts b/src/Decoder.ts
@@ -125,6 +125,15 @@ export class Decoder<ContextType> {
     return object;
   }
 
+  public *decodeMulti(buffer: ArrayLike<number> | BufferSource): Generator<unknown, void, unknown> {
+    this.reinitializeState();
+    this.setBuffer(buffer);
+
+    while (this.hasRemaining()) {
+      yield this.doDecodeSync();
+    }
+  }
+
   public async decodeAsync(stream: AsyncIterable<ArrayLike<number> | BufferSource>): Promise<unknown> {
     let decoded = false;
     let object: unknown;
diff --git a/src/decode.ts b/src/decode.ts
@@ -38,9 +38,10 @@ export type DecodeOptions<ContextType = undefined> = Readonly<
 export const defaultDecodeOptions: DecodeOptions = {};
 
 /**
- * It decodes a MessagePack-encoded buffer.
+ * It decodes a single MessagePack object in a buffer.
  *
- * This is a synchronous decoding function. See other variants for asynchronous decoding: `decodeAsync()`, `decodeStream()`, `decodeArrayStream()`.
+ * This is a synchronous decoding function.
+ * See other variants for asynchronous decoding: {@link decodeAsync()}, {@link decodeStream()}, or {@link decodeArrayStream()}.
  */
 export function decode<ContextType = undefined>(
   buffer: ArrayLike<number> | BufferSource,
@@ -57,3 +58,23 @@ export function decode<ContextType = undefined>(
   );
   return decoder.decode(buffer);
 }
+
+/**
+ * It decodes multiple MessagePack objects in a buffer.
+ * This is corresponding to {@link decodeMultiStream()}.
+ */
+export function decodeMulti<ContextType = undefined>(
+  buffer: ArrayLike<number> | BufferSource,
+  options: DecodeOptions<SplitUndefined<ContextType>> = defaultDecodeOptions as any,
+): Generator<unknown, void, unknown> {
+  const decoder = new Decoder(
+    options.extensionCodec,
+    (options as typeof options & { context: any }).context,
+    options.maxStrLength,
+    options.maxBinLength,
+    options.maxArrayLength,
+    options.maxMapLength,
+    options.maxExtLength,
+  );
+  return decoder.decodeMulti(buffer);
+}
diff --git a/src/decodeAsync.ts b/src/decodeAsync.ts
@@ -42,7 +42,7 @@ export function decodeArrayStream<ContextType>(
   return decoder.decodeArrayStream(stream);
 }
 
-export function decodeStream<ContextType>(
+export function decodeMultiStream<ContextType>(
   streamLike: ReadableStreamLike<ArrayLike<number> | BufferSource>,
   options: DecodeOptions<SplitUndefined<ContextType>> = defaultDecodeOptions as any,
 ) {
@@ -60,3 +60,13 @@ export function decodeStream<ContextType>(
 
   return decoder.decodeStream(stream);
 }
+
+/**
+ * @deprecated Use {@link decodeMultiStream()} instead.
+ */
+export function decodeStream<ContextType>(
+  streamLike: ReadableStreamLike<ArrayLike<number> | BufferSource>,
+  options: DecodeOptions<SplitUndefined<ContextType>> = defaultDecodeOptions as any,
+) {
+  return decodeMultiStream(streamLike, options);
+}
diff --git a/src/index.ts b/src/index.ts
@@ -5,13 +5,13 @@ export { encode };
 import type { EncodeOptions } from "./encode";
 export type { EncodeOptions };
 
-import { decode } from "./decode";
-export { decode };
+import { decode, decodeMulti } from "./decode";
+export { decode, decodeMulti };
 import type { DecodeOptions } from "./decode";
 export { DecodeOptions };
 
-import { decodeAsync, decodeArrayStream, decodeStream } from "./decodeAsync";
-export { decodeAsync, decodeArrayStream, decodeStream };
+import { decodeAsync, decodeArrayStream, decodeMultiStream, decodeStream } from "./decodeAsync";
+export { decodeAsync, decodeArrayStream, decodeMultiStream, decodeStream };
 
 import { Decoder } from "./Decoder";
 export { Decoder };
diff --git a/test/decodeMulti.test.ts b/test/decodeMulti.test.ts
@@ -0,0 +1,30 @@
+import assert from "assert";
+import { encode, decodeMulti } from "@msgpack/msgpack";
+
+describe("decodeMulti", () => {
+  it("decodes multiple objects in a single binary", () => {
+    const items = [
+      "foo",
+      10,
+      {
+        name: "bar",
+      },
+      [1, 2, 3],
+    ];
+
+    const encodedItems = items.map((item) => encode(item));
+    const encoded = new Uint8Array(encodedItems.reduce((p, c) => p + c.byteLength, 0));
+    let offset = 0;
+    for (const encodedItem of encodedItems) {
+      encoded.set(encodedItem, offset);
+      offset += encodedItem.byteLength;
+    }
+
+    const result: Array<unknown> = [];
+
+    for (const item of decodeMulti(encoded)) {
+      result.push(item);
+    }
+
+    assert.deepStrictEqual(result, items);
+  });});
diff --git a/test/decodeMultiStream.test.ts b/test/decodeMultiStream.test.ts
@@ -1,5 +1,5 @@
 import assert from "assert";
-import { encode, decodeStream } from "@msgpack/msgpack";
+import { encode, decodeMultiStream } from "@msgpack/msgpack";
 
 describe("decodeStream", () => {
   it("decodes stream", async () => {
@@ -20,14 +20,14 @@ describe("decodeStream", () => {
 
     const result: Array<unknown> = [];
 
-    for await (const item of decodeStream(createStream())) {
+    for await (const item of decodeMultiStream(createStream())) {
       result.push(item);
     }
 
     assert.deepStrictEqual(result, items);
   });
 
-  it("decodes stream with a single binary", async () => {
+  it("decodes multiple objects in a single binary stream", async () => {
     const items = [
       "foo",
       10,
@@ -51,7 +51,7 @@ describe("decodeStream", () => {
 
     const result: Array<unknown> = [];
 
-    for await (const item of decodeStream(createStream())) {
+    for await (const item of decodeMultiStream(createStream())) {
       result.push(item);
     }
 
