Merge branch 'develop' into feature/update-doc

Author: cyyeh

packages/core/src/lib/cache-layer/cacheLayerLoader.ts

@@ -62,6 +62,19 @@ export class CacheLayerLoader implements ICacheLayerLoader {
       if (!fs.existsSync(directory!)) {
         fs.mkdirSync(directory!, { recursive: true });
       }
+      // remove the files in other subfolder before export, cause we will not reuse cache files
+      const folderPath = path.resolve(
+        this.options.folderPath!,
+        templateName,
+        profile,
+        cacheTableName
+      );
+      const folders = fs
+        .readdirSync(folderPath)
+        .filter((file) =>
+          fs.statSync(path.resolve(folderPath, file)).isDirectory()
+        );
+      this.removeParquetFiles(folders, folderPath);
       // 1. export to cache files according to each schema set the cache value
       this.logger.debug(`Start to export to ${type} file in "${directory}"`);
       await dataSource.export({
@@ -97,4 +110,14 @@ export class CacheLayerLoader implements ICacheLayerLoader {
     const parquetFiles = files.filter((file) => /\.parquet$/.test(file));
     return parquetFiles;
   }
+
+  private removeParquetFiles(folders: string[], folderPath: string) {
+    folders.forEach((folder) => {
+      const directory = path.resolve(folderPath, folder);
+      const parquetFiles = this.getParquetFiles(directory);
+      parquetFiles.forEach((file) => {
+        fs.unlinkSync(path.resolve(directory, file));
+      });
+    });
+  }
 }

packages/core/src/lib/template-engine/built-in-extensions/cache/cacheTagRunner.ts

@@ -44,7 +44,9 @@ export class CacheTagRunner extends TagRunner {
     query = query
       .split(/\r?\n/)
       .filter((line) => line.trim().length > 0)
-      .join('\n');
+      .join('\n')
+      .replace(/--.*(?:\n|$)|\/\*[\s\S]*?\*\//g, ''); // remove single-line comments and multi-line comments
+
     // Set the default vulcan created cache table schema, so we could query the cache table directly, not need user to type schema in the SQL.
     query = `set schema=${vulcanCacheSchemaName};`.concat('\n').concat(query);
     // Create the builder which access "vulcan.cache" data source for cache layer query

packages/core/src/lib/template-engine/built-in-extensions/query-builder/reqTagRunner.ts

@@ -40,10 +40,12 @@ export class ReqTagRunner extends TagRunner {
     for (let index = 0; index < contentArgs.length; index++) {
       query += await contentArgs[index]();
     }
+
     query = query
       .split(/\r?\n/)
       .filter((line) => line.trim().length > 0)
-      .join('\n');
+      .join('\n')
+      .replace(/--.*(?:\n|$)|\/\*[\s\S]*?\*\//g, ''); // remove single-line comments and multi-line comments
 
     let builder: IDataQueryBuilder | undefined;
     // Replace to put the directly query cache builder to original query main builder of  "__wrapper__builder",

packages/core/test/cache-layer/cacheLayerLoader.spec.ts

@@ -169,6 +169,114 @@ describe('Test cache layer loader', () => {
     // Set 50s timeout to test cache loader export and load data
     50 * 10000
   );
+  it(
+    'Should remove the other parquet files after executing export',
+    async () => {
+      // Arrange
+      const templateName = 'template-1';
+      const cache = {
+        cacheTableName: 'employees',
+        sql: sinon.default.stub() as any,
+        profile: profiles[0].name,
+        folderSubpath: '2023',
+      } as CacheLayerInfo;
+      const { profile, cacheTableName, folderSubpath } = cache;
+      const dir = path.resolve(
+        folderPath,
+        templateName,
+        profile,
+        cacheTableName,
+        folderSubpath!
+      );
+      const loader = new CacheLayerLoader(options, stubFactory as any);
+      await loader.load(templateName, cache);
+      expect(fs.readdirSync(dir).length).toBeGreaterThan(0);
+
+      // Act
+      cache.folderSubpath = '2024';
+      await loader.load(templateName, cache);
+
+      // Assert
+      const newDir = path.resolve(
+        folderPath,
+        templateName,
+        profile,
+        cacheTableName,
+        '2024'!
+      );
+      expect(fs.readdirSync(dir).length).toEqual(0);
+      expect(fs.readdirSync(newDir).length).toBeGreaterThan(0);
+    },
+    // Set 50s timeout to test cache loader export and load data
+    50 * 10000
+  );
+  it(
+    'Should not remove files if parquet files were reused',
+    async () => {
+      const templateName = 'template-1';
+      const cache = {
+        cacheTableName: 'employees',
+        sql: sinon.default.stub() as any,
+        profile: profiles[0].name,
+        folderSubpath: '2023',
+      } as CacheLayerInfo;
+      // Arrange
+      const { profile, cacheTableName, folderSubpath } = cache;
+      const dir = path.resolve(
+        folderPath,
+        templateName,
+        profile,
+        cacheTableName,
+        folderSubpath!
+      );
+      const loader = new CacheLayerLoader(options, stubFactory as any);
+      await loader.load(templateName, cache);
+      const parquetFiles = fs.readdirSync(dir);
+
+      // Act
+      await loader.load(templateName, cache);
+
+      // Assert
+      // expect parquetFiles is the same
+      expect(fs.readdirSync(dir)).toEqual(parquetFiles);
+    },
+    // Set 50s timeout to test cache loader export and load data
+    50 * 10000
+  );
+
+  it(
+    'Should remove parquet files of its own folder.',
+    async () => {
+      // Arrange
+      const templateName = 'template-1';
+      const cache = {
+        cacheTableName: 'employees',
+        sql: sinon.default.stub() as any,
+        profile: profiles[0].name,
+        folderSubpath: '2023',
+      } as CacheLayerInfo;
+      const { profile, cacheTableName, folderSubpath } = cache;
+      const loader = new CacheLayerLoader(options, stubFactory as any);
+      await loader.load(templateName, cache);
+      const dirPath = path.resolve(
+        folderPath,
+        templateName,
+        profile,
+        cacheTableName,
+        folderSubpath!
+      );
+      expect(fs.readdirSync(dirPath).length).toBeGreaterThan(0);
+
+      // Act :load another cache table
+      cache.cacheTableName = 'another_employees';
+      await loader.load(templateName, cache);
+
+      // Assert
+      expect(fs.readdirSync(dirPath).length).toBeGreaterThan(0);
+    },
+    // Set 50s timeout to test cache loader export and load data
+    50 * 10000
+  );
 });
 
 async function createParquetFile(path: string, fileName: string) {

packages/core/test/template-engine/built-in-extensions/cache/cache.spec.ts

@@ -174,3 +174,48 @@ select count(*) as count from vulcan.user where user.id = {{ context.params.user
   expect(binding[0].get('$1')).toBe(`user-id`);
   expect(resultData).toEqual([{ count: 1 }]);
 });
+
+it('Extension should remove comments in sql statements', async () => {
+  // Arrange
+  const {
+    compiler,
+    loader,
+    builder,
+    executeTemplate,
+    getCreatedQueries,
+    getCreatedBinding,
+  } = await createTestCompiler();
+  const { compiledData } = await compiler.compile(`
+-- this is comment1
+/*
+this is multiline comments
+*/
+{% cache %} -- this is comment2
+select count(*) as count from vulcan.user where user.id = {{ context.params.userId }}; -- this is comment3
+{% endcache %} -- this is comment4
+-- this is comment5
+
+/*
+ this is multiline comments
+*/
+  `);
+  builder.value.onFirstCall().resolves({
+    getColumns: () => [],
+    getData: () => arrayToStream([{ count: 1 }]),
+  });
+  // Action
+  loader.setSource('test', compiledData);
+  const result = await executeTemplate('test', {
+    userId: 'user-id',
+  });
+  const resultData = await streamToArray(result.getData());
+  const queries = await getCreatedQueries();
+  const binding = await getCreatedBinding();
+
+  // Assert, cache tag runner add the "set schema=vulcan;" in the prefix sql statement to make query could get correct cache table of schema.
+  expect(queries[0].trim()).toBe(
+    `set schema=vulcan;\n select count(*) as count from vulcan.user where user.id = $1;`
+  );
+  expect(binding[0].get('$1')).toBe(`user-id`);
+  expect(resultData).toEqual([{ count: 1 }]);
+});

packages/core/test/template-engine/built-in-extensions/query-builder/builder.spec.ts

@@ -167,3 +167,53 @@ it('Extension should throw error when no profile defined', async () => {
     `No profile name found`
   );
 });
+
+it('Extension should remove comments in sql statements', async () => {
+  // Arrange
+  const {
+    compiler,
+    loader,
+    builder,
+    executeTemplate,
+    getCreatedQueries,
+    getCreatedBinding,
+  } = await createTestCompiler();
+  const { compiledData } = await compiler.compile(`
+-- this is comment1
+/*
+ this is multiline comments
+*/
+{% req userCount main %} -- this is comment2
+/*
+this is multiline comments
+*/
+-- this is comment3
+select count(*) as count from user where user.id = {{ context.params.userId }}; -- this is comment4
+-- this is comment5
+{% endreq %}
+
+/*
+ this is multiline comments
+*/
+
+-- this is comment6
+  `);
+  builder.value.onFirstCall().resolves({
+    getColumns: () => [],
+    getData: () => arrayToStream([{ count: 1 }]),
+  });
+  // Action
+  loader.setSource('test', compiledData);
+  const result = await executeTemplate('test', {
+    userId: 'user-id',
+  });
+  const resultData = await streamToArray(result.getData());
+  const queries = await getCreatedQueries();
+  const binding = await getCreatedBinding();
+  // Assert
+  expect(queries[0].trim()).toBe(
+    `select count(*) as count from user where user.id = $1;`
+  );
+  expect(binding[0].get('$1')).toBe(`user-id`);
+  expect(resultData).toEqual([{ count: 1 }]);
+});