feat: add getSimplifiedSchema helper which returns a schema without any metadata COMPASS-6979 (#198)

Author: Anemy

README.md

@@ -55,7 +55,7 @@ below accordingly).
 3. Create a new file `parse-schema.js` and paste in the following code:
 
     ```javascript
-    const parseSchema = require('mongodb-schema');
+    const { parseSchema } = require('mongodb-schema');
     const { MongoClient } = require('mongodb');
 
     const dbName = 'test';

src/index.ts

@@ -3,6 +3,7 @@ import { Readable, PassThrough } from 'stream';
 import { pipeline } from 'stream/promises';
 
 import stream from './stream';
+import type { ParseStreamOptions } from './stream';
 import { SchemaAnalyzer } from './schema-analyzer';
 import type {
   ArraySchemaType,
@@ -13,7 +14,13 @@ import type {
   SchemaType,
   Schema,
   SchemaField,
-  SchemaParseOptions
+  SchemaParseOptions,
+  SimplifiedSchemaBaseType,
+  SimplifiedSchemaArrayType,
+  SimplifiedSchemaDocumentType,
+  SimplifiedSchemaType,
+  SimplifiedSchemaField,
+  SimplifiedSchema
 } from './schema-analyzer';
 import * as schemaStats from './stats';
 
@@ -42,43 +49,47 @@ function getStreamSource(
   return streamSource;
 }
 
-/**
- * Convenience shortcut for parsing schemas. Accepts an array, stream or
- * MongoDB cursor object to parse documents` from.
- */
-async function parseSchema(
+async function schemaStream(
   source: Document[] | MongoDBCursor | Readable,
-  options?: SchemaParseOptions
-): Promise<Schema> {
-  // Shift parameters if no options are specified.
-  if (typeof options === 'undefined') {
-    options = {};
-  }
-
+  options?: ParseStreamOptions
+) {
   const streamSource = getStreamSource(source);
 
   const dest = new PassThrough({ objectMode: true });
   await pipeline(streamSource, stream(options), dest);
   for await (const result of dest) {
     return result;
   }
-  throw new Error('unreachable'); // `dest` always emits one doc.
+  throw new Error('unreachable'); // `dest` always emits exactly one doc.
+}
+
+/**
+ * Convenience shortcut for parsing schemas. Accepts an array, stream or
+ * MongoDB cursor object to parse documents` from.
+ */
+async function parseSchema(
+  source: Document[] | MongoDBCursor | Readable,
+  options?: SchemaParseOptions
+): Promise<Schema> {
+  return await schemaStream(source, options);
 }
 
 // Convenience shortcut for getting schema paths.
 async function getSchemaPaths(
   source: Document[] | MongoDBCursor | Readable
 ): Promise<string[][]> {
-  const streamSource = getStreamSource(source);
-
-  const dest = new PassThrough({ objectMode: true });
-  await pipeline(streamSource, stream({
+  return await schemaStream(source, {
     schemaPaths: true
-  }), dest);
-  for await (const result of dest) {
-    return result;
-  }
-  throw new Error('unreachable'); // `dest` always emits one doc.
+  });
+}
+
+// Convenience shortcut for getting the simplified schema.
+async function getSimplifiedSchema(
+  source: Document[] | MongoDBCursor | Readable
+): Promise<SimplifiedSchema> {
+  return await schemaStream(source, {
+    simplifiedSchema: true
+  });
 }
 
 export default parseSchema;
@@ -92,12 +103,20 @@ export type {
   SchemaType,
   Schema,
   SchemaField,
-  SchemaParseOptions
+  SchemaParseOptions,
+  SimplifiedSchemaBaseType,
+  SimplifiedSchemaArrayType,
+  SimplifiedSchemaDocumentType,
+  SimplifiedSchemaType,
+  SimplifiedSchemaField,
+  SimplifiedSchema
 };
 
 export {
   stream,
+  parseSchema,
   getSchemaPaths,
+  getSimplifiedSchema,
   SchemaAnalyzer,
   schemaStats
 };

src/schema-analyzer.ts

@@ -270,6 +270,55 @@ function schemaToPaths(
   return paths;
 }
 
+export type SimplifiedSchemaBaseType = {
+  bsonType: SchemaBSONType;
+}
+export type SimplifiedSchemaArrayType = SimplifiedSchemaBaseType & {
+  bsonType: 'Array';
+  // eslint-disable-next-line no-use-before-define
+  types: SimplifiedSchemaType[];
+}
+export type SimplifiedSchemaDocumentType = SimplifiedSchemaBaseType & {
+  bsonType: 'Document';
+  // eslint-disable-next-line no-use-before-define
+  fields: SimplifiedSchema;
+}
+export type SimplifiedSchemaType = SimplifiedSchemaBaseType | SimplifiedSchemaArrayType | SimplifiedSchemaDocumentType;
+export type SimplifiedSchemaField = {
+  types: SimplifiedSchemaType[];
+};
+export type SimplifiedSchema = {
+  [fieldName: string]: SimplifiedSchemaField
+}
+
+function simplifiedSchema(fields: SchemaAnalysisFieldsMap): SimplifiedSchema {
+  function finalizeSchemaFieldTypes(types: SchemaAnalysisFieldTypes): SimplifiedSchemaType[] {
+    return Object.values(types).map((type) => {
+      return {
+        bsonType: type.bsonType, // Note: `Object` is replaced with `Document`.
+        ...(isArrayType(type) ? {
+          types: finalizeSchemaFieldTypes(type.types)
+        } : {}),
+        ...(isDocumentType(type) ? { fields: finalizeDocumentFieldSchema(type.fields) } : {})
+      };
+    });
+  }
+
+  function finalizeDocumentFieldSchema(fieldMap: SchemaAnalysisFieldsMap): SimplifiedSchema {
+    const fieldSchema: SimplifiedSchema = {};
+    Object.values(fieldMap).forEach((field: SchemaAnalysisField) => {
+      const fieldTypes = finalizeSchemaFieldTypes(field.types);
+
+      fieldSchema[field.name] = {
+        types: fieldTypes
+      };
+    });
+    return fieldSchema;
+  }
+
+  return finalizeDocumentFieldSchema(fields);
+}
+
 function cropStringAt10kCharacters(value: string) {
   return value.charCodeAt(10000 - 1) === value.codePointAt(10000 - 1)
     ? value.slice(0, 10000)
@@ -523,4 +572,11 @@ export class SchemaAnalyzer {
   getSchemaPaths(): string[][] {
     return schemaToPaths(this.schemaAnalysisRoot.fields);
   }
+
+  /**
+   * Returns a simplified schema result, this has no type metadata.
+   */
+  getSimplifiedSchema(): SimplifiedSchema {
+    return simplifiedSchema(this.schemaAnalysisRoot.fields);
+  }
 }

src/stream.ts

@@ -7,14 +7,19 @@ import type {
 import { SchemaAnalyzer } from './schema-analyzer';
 import type { SchemaParseOptions } from './schema-analyzer';
 
+export type ParseStreamOptions = SchemaParseOptions & {
+  simplifiedSchema?: boolean,
+  schemaPaths?: boolean;
+};
+
 export class ParseStream extends Duplex {
   analyzer: SchemaAnalyzer;
+  options: ParseStreamOptions;
   schemaPaths = false;
-  constructor(options?: SchemaParseOptions & {
-    schemaPaths?: boolean;
-  }) {
+
+  constructor(options?: ParseStreamOptions) {
     super({ objectMode: true });
-    this.schemaPaths = !!options?.schemaPaths;
+    this.options = options || {};
     this.analyzer = new SchemaAnalyzer(options);
   }
 
@@ -27,8 +32,10 @@ export class ParseStream extends Duplex {
   _read() {}
 
   _final(cb: () => void) {
-    if (this.schemaPaths) {
+    if (this.options.schemaPaths) {
       this.push(this.analyzer.getSchemaPaths());
+    } else if (this.options.simplifiedSchema) {
+      this.push(this.analyzer.getSimplifiedSchema());
     } else {
       this.push(this.analyzer.getResult());
     }
@@ -37,8 +44,6 @@ export class ParseStream extends Duplex {
   }
 }
 
-export default function makeParseStream(options?: SchemaParseOptions & {
-    schemaPaths?: boolean;
-  }) {
+export default function makeParseStream(options?: ParseStreamOptions) {
   return new ParseStream(options);
 }

test/all-bson-types-fixture.ts

@@ -0,0 +1,57 @@
+import {
+  BSONRegExp,
+  Binary,
+  Code,
+  DBRef,
+  Decimal128,
+  Double,
+  Int32,
+  Long,
+  MaxKey,
+  MinKey,
+  ObjectId,
+  Timestamp,
+  UUID,
+  BSONSymbol
+} from 'bson';
+
+export const allBSONTypesDoc = {
+  _id: new ObjectId('642d766b7300158b1f22e972'),
+  double: new Double(1.2), // Double, 1, double
+  string: 'Hello, world!', // String, 2, string
+  object: { key: 'value' }, // Object, 3, object
+  array: [1, 2, 3], // Array, 4, array
+  binData: new Binary(Buffer.from([1, 2, 3])), // Binary data, 5, binData
+  // Undefined, 6, undefined (deprecated)
+  objectId: new ObjectId('642d766c7300158b1f22e975'), // ObjectId, 7, objectId
+  boolean: true, // Boolean, 8, boolean
+  date: new Date('2023-04-05T13:25:08.445Z'), // Date, 9, date
+  null: null, // Null, 10, null
+  regex: new BSONRegExp('pattern', 'i'), // Regular Expression, 11, regex
+  // DBPointer, 12, dbPointer (deprecated)
+  javascript: new Code('function() {}'), // JavaScript, 13, javascript
+  symbol: new BSONSymbol('symbol'), // Symbol, 14, symbol (deprecated)
+  javascriptWithScope: new Code('function() {}', { foo: 1, bar: 'a' }), // JavaScript code with scope 15 "javascriptWithScope" Deprecated in MongoDB 4.4.
+  int: new Int32(12345), // 32-bit integer, 16, "int"
+  timestamp: new Timestamp(new Long('7218556297505931265')), // Timestamp, 17, timestamp
+  long: new Long('123456789123456789'), // 64-bit integer, 18, long
+  decimal: new Decimal128(
+    Buffer.from([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16])
+  ), // Decimal128, 19, decimal
+  minKey: new MinKey(), // Min key, -1, minKey
+  maxKey: new MaxKey(), // Max key, 127, maxKey
+
+  binaries: {
+    generic: new Binary(Buffer.from([1, 2, 3]), 0), // 0
+    functionData: new Binary('//8=', 1), // 1
+    binaryOld: new Binary('//8=', 2), // 2
+    uuidOld: new Binary('c//SZESzTGmQ6OfR38A11A==', 3), // 3
+    uuid: new UUID('AAAAAAAA-AAAA-4AAA-AAAA-AAAAAAAAAAAA'), // 4
+    md5: new Binary('c//SZESzTGmQ6OfR38A11A==', 5), // 5
+    encrypted: new Binary('c//SZESzTGmQ6OfR38A11A==', 6), // 6
+    compressedTimeSeries: new Binary('c//SZESzTGmQ6OfR38A11A==', 7), // 7
+    custom: new Binary('//8=', 128) // 128
+  },
+
+  dbRef: new DBRef('namespace', new ObjectId('642d76b4b7ebfab15d3c4a78')) // not actually a separate type, just a convention
+};

test/all-bson-types.test.ts

@@ -1,71 +1,13 @@
-import {
-  BSONRegExp,
-  Binary,
-  Code,
-  DBRef,
-  Decimal128,
-  Double,
-  Int32,
-  Long,
-  MaxKey,
-  MinKey,
-  ObjectId,
-  Timestamp,
-  UUID,
-  BSONSymbol
-} from 'bson';
 import assert from 'assert';
 
 import type { Schema } from '../src/schema-analyzer';
 import getSchema from '../src';
-
-const allBsonTypes = [
-  {
-    _id: new ObjectId('642d766b7300158b1f22e972'),
-    double: new Double(1.2), // Double, 1, double
-    string: 'Hello, world!', // String, 2, string
-    object: { key: 'value' }, // Object, 3, object
-    array: [1, 2, 3], // Array, 4, array
-    binData: new Binary(Buffer.from([1, 2, 3])), // Binary data, 5, binData
-    // Undefined, 6, undefined (deprecated)
-    objectId: new ObjectId('642d766c7300158b1f22e975'), // ObjectId, 7, objectId
-    boolean: true, // Boolean, 8, boolean
-    date: new Date('2023-04-05T13:25:08.445Z'), // Date, 9, date
-    null: null, // Null, 10, null
-    regex: new BSONRegExp('pattern', 'i'), // Regular Expression, 11, regex
-    // DBPointer, 12, dbPointer (deprecated)
-    javascript: new Code('function() {}'), // JavaScript, 13, javascript
-    symbol: new BSONSymbol('symbol'), // Symbol, 14, symbol (deprecated)
-    javascriptWithScope: new Code('function() {}', { foo: 1, bar: 'a' }), // JavaScript code with scope 15 "javascriptWithScope" Deprecated in MongoDB 4.4.
-    int: new Int32(12345), // 32-bit integer, 16, "int"
-    timestamp: new Timestamp(new Long('7218556297505931265')), // Timestamp, 17, timestamp
-    long: new Long('123456789123456789'), // 64-bit integer, 18, long
-    decimal: new Decimal128(
-      Buffer.from([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16])
-    ), // Decimal128, 19, decimal
-    minKey: new MinKey(), // Min key, -1, minKey
-    maxKey: new MaxKey(), // Max key, 127, maxKey
-
-    binaries: {
-      generic: new Binary(Buffer.from([1, 2, 3]), 0), // 0
-      functionData: new Binary('//8=', 1), // 1
-      binaryOld: new Binary('//8=', 2), // 2
-      uuidOld: new Binary('c//SZESzTGmQ6OfR38A11A==', 3), // 3
-      uuid: new UUID('AAAAAAAA-AAAA-4AAA-AAAA-AAAAAAAAAAAA'), // 4
-      md5: new Binary('c//SZESzTGmQ6OfR38A11A==', 5), // 5
-      encrypted: new Binary('c//SZESzTGmQ6OfR38A11A==', 6), // 6
-      compressedTimeSeries: new Binary('c//SZESzTGmQ6OfR38A11A==', 7), // 7
-      custom: new Binary('//8=', 128) // 128
-    },
-
-    dbRef: new DBRef('namespace', new ObjectId('642d76b4b7ebfab15d3c4a78')) // not actually a separate type, just a convention
-  }
-];
+import { allBSONTypesDoc } from './all-bson-types-fixture';
 
 describe('using a document with all bson types', function() {
   let schema: Schema;
   before(async function() {
-    schema = await getSchema(allBsonTypes);
+    schema = await getSchema([allBSONTypesDoc]);
   });
 
   it('contains all of the types', function() {