feat: add getSchemaPaths helper (#192)

Author: Anemy

src/index.ts

@@ -1,6 +1,6 @@
 import type { AggregationCursor, Document, FindCursor } from 'mongodb';
-import { pipeline as callbackPipeline, Readable, PassThrough } from 'stream';
-import { promisify } from 'util';
+import { Readable, PassThrough } from 'stream';
+import { pipeline } from 'stream/promises';
 
 import stream from './stream';
 import { SchemaAnalyzer } from './schema-analyzer';
@@ -19,39 +19,62 @@ import * as schemaStats from './stats';
 
 type MongoDBCursor = AggregationCursor | FindCursor;
 
+function getStreamSource(
+  source: Document[] | MongoDBCursor | Readable
+): Readable {
+  let streamSource: Readable;
+  if ('stream' in source) {
+    // MongoDB Cursor.
+    streamSource = source.stream();
+  } else if ('pipe' in source) {
+    // Document stream.
+    streamSource = source;
+  } else if (Array.isArray(source)) {
+    // Array of documents.
+    streamSource = Readable.from(source);
+  } else {
+    throw new Error(
+      'Unknown input type for `docs`. Must be an array, ' +
+        'stream or MongoDB Cursor.'
+    );
+  }
+
+  return streamSource;
+}
+
 /**
  * Convenience shortcut for parsing schemas. Accepts an array, stream or
  * MongoDB cursor object to parse documents` from.
  */
 async function parseSchema(
-  docs: Document[] | MongoDBCursor | Readable,
+  source: Document[] | MongoDBCursor | Readable,
   options?: SchemaParseOptions
 ): Promise<Schema> {
   // Shift parameters if no options are specified.
   if (typeof options === 'undefined') {
     options = {};
   }
 
-  let src: Readable;
-  if ('stream' in docs) {
-    // MongoDB Cursor.
-    src = docs.stream();
-  } else if ('pipe' in docs) {
-    // Document stream.
-    src = docs;
-  } else if (Array.isArray(docs)) {
-    // Array of documents.
-    src = Readable.from(docs);
-  } else {
-    throw new Error(
-      'Unknown input type for `docs`. Must be an array, ' +
-        'stream or MongoDB Cursor.'
-    );
+  const streamSource = getStreamSource(source);
+
+  const dest = new PassThrough({ objectMode: true });
+  await pipeline(streamSource, stream(options), dest);
+  for await (const result of dest) {
+    return result;
   }
+  throw new Error('unreachable'); // `dest` always emits one doc.
+}
+
+// Convenience shortcut for getting schema paths.
+async function getSchemaPaths(
+  source: Document[] | MongoDBCursor | Readable
+): Promise<string[][]> {
+  const streamSource = getStreamSource(source);
 
   const dest = new PassThrough({ objectMode: true });
-  const pipeline = promisify(callbackPipeline);
-  await pipeline(src, stream(options), dest);
+  await pipeline(streamSource, stream({
+    schemaPaths: true
+  }), dest);
   for await (const result of dest) {
     return result;
   }
@@ -74,6 +97,7 @@ export type {
 
 export {
   stream,
+  getSchemaPaths,
   SchemaAnalyzer,
   schemaStats
 };

src/schema-analyzer.ts

@@ -225,6 +225,46 @@ function isDocumentType(type: SchemaAnalysisType): type is SchemaAnalysisDocumen
   return (type as SchemaAnalysisDocumentType).name === 'Document';
 }
 
+/**
+ * Recursively extracts all of the schema field paths as string arrays.
+ */
+function schemaToPaths(
+  fields: SchemaAnalysisFieldsMap,
+  parent: string[] = []
+): string[][] {
+  const paths: string[][] = [];
+
+  for (const field of Object.values(fields)) {
+    const path = [...parent, field.name];
+    paths.push(path);
+
+    // Recurse on doc.
+    const doc = Object.values(field.types).find((f) => f.bsonType === 'Document') as
+      | SchemaAnalysisDocumentType
+      | undefined;
+
+    if (doc) {
+      paths.push(...schemaToPaths(doc.fields, path));
+    }
+
+    // Recurse on array.
+    const array = Object.values(field.types).find((f) => f.bsonType === 'Array') as
+      | SchemaAnalysisArrayType
+      | undefined;
+    if (array) {
+      const arrayDoc = Object.values(array.types).find((f) => f.bsonType === 'Document') as
+        | SchemaAnalysisDocumentType
+        | undefined;
+
+      if (arrayDoc) {
+        paths.push(...schemaToPaths(arrayDoc.fields, path));
+      }
+    }
+  }
+
+  return paths;
+}
+
 function cropStringAt10kCharacters(value: string) {
   return value.charCodeAt(10000 - 1) === value.codePointAt(10000 - 1)
     ? value.slice(0, 10000)
@@ -474,4 +514,8 @@ export class SchemaAnalyzer {
     this.finalized = true;
     return this.schemaResult;
   }
+
+  getSchemaPaths(): string[][] {
+    return schemaToPaths(this.schemaAnalysisRoot.fields);
+  }
 }

src/stream.ts

@@ -9,8 +9,12 @@ import type { SchemaParseOptions } from './schema-analyzer';
 
 export class ParseStream extends Duplex {
   analyzer: SchemaAnalyzer;
-  constructor(options?: SchemaParseOptions) {
+  schemaPaths = false;
+  constructor(options?: SchemaParseOptions & {
+    schemaPaths?: boolean;
+  }) {
     super({ objectMode: true });
+    this.schemaPaths = !!options?.schemaPaths;
     this.analyzer = new SchemaAnalyzer(options);
   }
 
@@ -23,12 +27,18 @@ export class ParseStream extends Duplex {
   _read() {}
 
   _final(cb: () => void) {
-    this.push(this.analyzer.getResult());
+    if (this.schemaPaths) {
+      this.push(this.analyzer.getSchemaPaths());
+    } else {
+      this.push(this.analyzer.getResult());
+    }
     this.push(null);
     cb();
   }
 }
 
-export default function makeParseStream(options?: SchemaParseOptions) {
+export default function makeParseStream(options?: SchemaParseOptions & {
+    schemaPaths?: boolean;
+  }) {
   return new ParseStream(options);
 }

test/get-schema-paths.test.ts

@@ -0,0 +1,111 @@
+import assert from 'assert';
+import type { Document } from 'bson';
+
+import { getSchemaPaths } from '../src';
+
+describe('getSchemaPaths', function() {
+  let schemaPaths: string[][];
+
+  describe('with fields with dots in them', function() {
+    const docs = [
+      {
+        pineapple: {
+          orange: {
+            apple: 1
+          },
+          'orange.with.dot': {
+            bah: 2
+          }
+        }
+      }
+    ];
+
+    before(async function() {
+      schemaPaths = await getSchemaPaths(docs);
+    });
+
+    it('returns an array of the fields', function() {
+      assert.deepEqual(schemaPaths, [
+        ['pineapple'],
+        ['pineapple', 'orange'],
+        ['pineapple', 'orange', 'apple'],
+        ['pineapple', 'orange.with.dot'],
+        ['pineapple', 'orange.with.dot', 'bah']
+      ]);
+    });
+  });
+
+  describe('with multiple documents with different fields', function() {
+    const docs = [
+      {
+        pineapple: {
+          orange: {
+            apple: 1
+          }
+        }
+      },
+      {
+        pineapple: {
+          orange: 'ok'
+        }
+      },
+      {
+        pineapple: ['test', '123'],
+        clementine: false
+      }
+    ];
+
+    before(async function() {
+      schemaPaths = await getSchemaPaths(docs);
+    });
+
+    it('returns all of the field paths', function() {
+      assert.deepEqual(schemaPaths, [
+        ['pineapple'],
+        ['pineapple', 'orange'],
+        ['pineapple', 'orange', 'apple'],
+        ['clementine']
+      ]);
+    });
+  });
+
+  describe('with nested array documents', function() {
+    const docs = [
+      {
+        orangutan: [{
+          tuatara: 'yes',
+          lizard: {
+            snakes: false,
+            birds: false
+          }
+        }]
+      }
+    ];
+
+    before(async function() {
+      schemaPaths = await getSchemaPaths(docs);
+    });
+
+    it('returns all of the field paths', function() {
+      assert.deepEqual(schemaPaths, [
+        ['orangutan'],
+        ['orangutan', 'tuatara'],
+        ['orangutan', 'lizard'],
+        ['orangutan', 'lizard', 'snakes'],
+        ['orangutan', 'lizard', 'birds']
+      ]);
+    });
+  });
+
+  describe('with no documents', function() {
+    const docs: Document[] = [];
+
+    before(async function() {
+      schemaPaths = await getSchemaPaths(docs);
+    });
+
+    it('returns no paths', function() {
+      assert.deepEqual(schemaPaths, []);
+    });
+  });
+});