Skip to content

Commit 91a542f

Browse files
committed
Merge branch 'develop' into feature/update-doc
2 parents 4d9feee + 56f1cdb commit 91a542f

File tree

51 files changed

+1080
-122
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+1080
-122
lines changed

labs/playground1/Makefile

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,13 @@
33

44
# start vulcan server (default goal)
55
start: build test-data/moma.db ../../node_modules
6-
@vulcan start
6+
@vulcan start --watch
77

88
# build the required packages
99
build: pkg-core pkg-build pkg-serve pkg-catalog-server pkg-cli pkg-extension-driver-duckdb \
1010
pkg-extension-authenticator-canner pkg-extension-driver-canner \
1111
pkg-extension-driver-clickhouse pkg-extension-driver-bq \
12-
pkg-extension-huggingface
12+
pkg-extension-huggingface pkg-extension-api-caller
1313

1414

1515
# build for core pakge
@@ -65,7 +65,7 @@ pkg-extension-driver-canner: ../../node_modules
6565
mkdir -p ./labs/playground1/node_modules/@vulcan-sql; \
6666
rm -rf ./labs/playground1/node_modules/@vulcan-sql/extension-driver-canner; \
6767
cp -R ./dist/packages/extension-driver-canner ./labs/playground1/node_modules/@vulcan-sql; \
68-
68+
6969
pkg-extension-driver-clickhouse: ../../node_modules
7070
@cd ../..; \
7171
yarn nx build extension-driver-clickhouse; \
@@ -89,8 +89,12 @@ pkg-extension-huggingface: ../../node_modules
8989
cp -R ./dist/packages/extension-huggingface ./labs/playground1/node_modules/@vulcan-sql; \
9090
cp -R ./packages/extension-huggingface/node_modules ./labs/playground1
9191

92-
93-
92+
pkg-extension-api-caller: ../../node_modules
93+
@cd ../..; \
94+
yarn nx build extension-api-caller; \
95+
mkdir -p ./labs/playground1/node_modules/@vulcan-sql; \
96+
rm -rf ./labs/playground1/node_modules/@vulcan-sql/extension-api-caller; \
97+
cp -R ./dist/packages/extension-api-caller ./labs/playground1/node_modules/@vulcan-sql
9498

9599
# build and install for cli pakge
96100
pkg-cli: ../../node_modules
Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
select
22
*
33
from "artists"
4-
where
4+
where
55
ConstituentID = {{ context.params.id }}
6+
{% if contex.params.begin_date %}
7+
and BeginDate = {{ context.params.begin_date }}
8+
{% endif %}

packages/build/package.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"name": "@vulcan-sql/build",
33
"description": "VulcanSQL package for building projects",
4-
"version": "0.8.1",
4+
"version": "0.9.0",
55
"type": "commonjs",
66
"publishConfig": {
77
"access": "public"
@@ -22,6 +22,6 @@
2222
},
2323
"license": "MIT",
2424
"peerDependencies": {
25-
"@vulcan-sql/core": "~0.8.1-0"
25+
"@vulcan-sql/core": "~0.9.0-0"
2626
}
27-
}
27+
}

packages/catalog-server/package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"name": "@vulcan-sql/catalog-server",
33
"description": "Catalog server for VulcanSQL",
4-
"version": "0.8.1",
4+
"version": "0.9.0",
55
"publishConfig": {
66
"access": "public"
77
},
@@ -21,4 +21,4 @@
2121
"url": "https://github.com/Canner/vulcan-sql.git"
2222
},
2323
"license": "MIT"
24-
}
24+
}

packages/cli/package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"name": "@vulcan-sql/cli",
33
"description": "CLI tools for VulcanSQL",
4-
"version": "0.8.1",
4+
"version": "0.9.0",
55
"type": "commonjs",
66
"bin": {
77
"vulcan": "./src/index.js"
@@ -24,4 +24,4 @@
2424
"url": "https://github.com/Canner/vulcan-sql.git"
2525
},
2626
"license": "MIT"
27-
}
27+
}

packages/core/package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
{
22
"name": "@vulcan-sql/core",
33
"description": "Core package of VulcanSQL",
4-
"version": "0.8.1",
4+
"version": "0.9.0",
55
"type": "commonjs",
66
"publishConfig": {
77
"access": "public"
@@ -21,4 +21,4 @@
2121
"url": "https://github.com/Canner/vulcan-sql.git"
2222
},
2323
"license": "MIT"
24-
}
24+
}

packages/core/src/containers/modules/dataSource.ts

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,11 @@ import {
88
import { Profile } from '../../models/profile';
99
import { ClassType } from '../../lib/utils/module';
1010
import { ConfigurationError } from '@vulcan-sql/core/utils';
11-
import { ICacheLayerOptions, cacheProfileName } from '@vulcan-sql/core/models';
11+
import {
12+
ICacheLayerOptions,
13+
cacheLayerPersistentFileName,
14+
cacheProfileName,
15+
} from '@vulcan-sql/core/models';
1216
import 'reflect-metadata';
1317

1418
export const dataSourceModule = (
@@ -26,6 +30,7 @@ export const dataSourceModule = (
2630
type: options.loader!.toLocaleLowerCase(),
2731
// allow '*' to make every user request could use the cache-layer data source.
2832
allow: '*',
33+
connection: { ['persistent-path']: cacheLayerPersistentFileName },
2934
} as Profile);
3035
}
3136

packages/core/src/lib/cache-layer/cacheLayerLoader.ts

Lines changed: 30 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -43,31 +43,40 @@ export class CacheLayerLoader implements ICacheLayerLoader {
4343
templateName: string,
4444
cache: CacheLayerInfo
4545
): Promise<void> {
46-
const { cacheTableName, sql, profile, indexes } = cache;
46+
const { cacheTableName, sql, profile, indexes, folderSubpath } = cache;
4747
const type = this.options.type!;
4848
const dataSource = this.dataSourceFactory(profile);
4949

5050
// generate directory for cache file path to export
5151
// format => [folderPath]/[schema.templateSource]/[profileName]/[cacheTableName]]/[timestamp]
52+
const subpath = folderSubpath || moment.utc().format('YYYYMMDDHHmmss');
5253
const directory = path.resolve(
5354
this.options.folderPath!,
5455
templateName,
5556
profile,
5657
cacheTableName,
57-
moment.utc().format('YYYYMMDDHHmmss')
58+
subpath
5859
);
59-
60-
if (!fs.existsSync(directory!))
61-
fs.mkdirSync(directory!, { recursive: true });
62-
63-
// 1. export to cache files according to each schema set the cache value
64-
this.logger.debug(`Start to export to ${type} file in "${directory}"`);
65-
await dataSource.export({
66-
sql,
67-
directory,
68-
profileName: profile,
69-
type,
70-
});
60+
const parquetFiles = this.getParquetFiles(directory);
61+
if (!parquetFiles.length) {
62+
if (!fs.existsSync(directory!)) {
63+
fs.mkdirSync(directory!, { recursive: true });
64+
}
65+
// 1. export to cache files according to each schema set the cache value
66+
this.logger.debug(`Start to export to ${type} file in "${directory}"`);
67+
await dataSource.export({
68+
sql,
69+
directory,
70+
profileName: profile,
71+
type,
72+
});
73+
} else {
74+
this.logger.debug(
75+
`Parquet file \n ${parquetFiles.join(
76+
'\n '
77+
)} found in ${directory}, skip export`
78+
);
79+
}
7180
this.logger.debug(`Start to load ${cacheTableName} in "${directory}"`);
7281
// 2. load the files to cache data source
7382
await this.cacheStorage.import({
@@ -81,4 +90,11 @@ export class CacheLayerLoader implements ICacheLayerLoader {
8190
indexes,
8291
});
8392
}
93+
94+
private getParquetFiles(directory: string): string[] {
95+
if (!directory || !fs.existsSync(directory)) return [];
96+
const files = fs.readdirSync(directory);
97+
const parquetFiles = files.filter((file) => /\.parquet$/.test(file));
98+
return parquetFiles;
99+
}
84100
}
Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
import * as fs from 'fs';
2+
import * as path from 'path';
3+
import { isEmpty } from 'lodash';
4+
5+
interface ConcurrentPerformanceRecord {
6+
[key: string]: [{ group?: string; diff?: number }];
7+
}
8+
let is_analysis = false;
9+
let performanceRecord: ConcurrentPerformanceRecord = {};
10+
let keyStatistics: Record<
11+
string,
12+
{
13+
min?: number;
14+
max?: number;
15+
avg?: number;
16+
median?: number;
17+
p90?: number;
18+
}
19+
> = {};
20+
/**
21+
* This is a performance analysis tool for concurrent tasks
22+
* You can use it to collect the start and end time of a task, the collected data with the same key will be summarized
23+
* the summarzied report contains the min, max, avg, median, p90 of the task
24+
* When the code snippet is executed, the performance analysis tool will automatically collect the start and end time of the task
25+
*
26+
* example:
27+
* const start = Date.now();
28+
* await fn_to_measure()
29+
* const end = Date.now();
30+
* PerformanceAnalysis.collect('fn_to_measure', start, end)
31+
*
32+
* You can choose when to summarize the performance data
33+
* for example, you can summarize the performance data before server closed
34+
*
35+
* public async close() {
36+
if (this.servers) {
37+
... close server
38+
}
39+
PerformanceAnalysis.count();
40+
}
41+
*
42+
* Note: If you want to view the performance by each API call, you can use k6 or you can specify the group name when collecting the performance data
43+
* and implement another count & writePerformanceReport funtion to summarize the performance data by group name
44+
*
45+
*/
46+
export class PerformanceAnalysis {
47+
public static collect(
48+
key: string,
49+
start: number,
50+
end: number,
51+
group?: string
52+
) {
53+
if (!start || !end) {
54+
throw new Error(
55+
`should provide start and end time when doing performance analysis task "${key}"`
56+
);
57+
}
58+
if (!performanceRecord[key]) {
59+
performanceRecord[key] = [] as any;
60+
}
61+
const diff = end - start;
62+
performanceRecord[key].push({ group, diff });
63+
if (process.env['PRINT_COLLECTION']) {
64+
console.log(
65+
`${key}: collected, start: ${start}, end: ${end}, diff: ${diff}`
66+
);
67+
}
68+
}
69+
70+
public static count(): boolean {
71+
// sort by time diff
72+
if (isEmpty(performanceRecord)) {
73+
console.log('performanceRecord is empty');
74+
return false;
75+
}
76+
Object.values(performanceRecord).map((records) => {
77+
records.sort((a, b) => {
78+
return <number>a.diff - <number>b.diff;
79+
});
80+
});
81+
// count statistics
82+
Object.entries(performanceRecord).map(([key, records]) => {
83+
const count = records.length;
84+
const min = records[0].diff;
85+
const max = records[count - 1].diff;
86+
const avg =
87+
records.reduce((acc, cur) => {
88+
return acc + <number>cur.diff;
89+
}, 0) / count;
90+
const median = records[Math.floor(count / 2)].diff;
91+
const p90 = records[Math.floor(count * 0.9)].diff;
92+
keyStatistics[key] = { min, max, avg, median, p90 };
93+
});
94+
return true;
95+
}
96+
97+
public static getStatistic(key: string): any {
98+
return keyStatistics[key];
99+
}
100+
101+
public static clean = () => {
102+
performanceRecord = {};
103+
keyStatistics = {};
104+
};
105+
106+
// write to txt file
107+
public static writePerformanceReport() {
108+
const filePath = path.join('./performanceRecord.txt');
109+
// print current date, time as human readable format
110+
fs.appendFileSync(filePath, `------${new Date().toLocaleString()}\n`);
111+
for (const key of Object.keys(keyStatistics)) {
112+
fs.appendFileSync(filePath, `${key}\n`);
113+
let staticLine = '';
114+
if (keyStatistics[key]) {
115+
const statics = keyStatistics[key];
116+
Object.entries(statics).map(([k, v]) => {
117+
staticLine += `${k}: ${v}, `;
118+
});
119+
fs.appendFileSync(filePath, `${staticLine}\n`);
120+
}
121+
}
122+
fs.appendFileSync(filePath, `------\n`);
123+
}
124+
}
125+
126+
export function getAnalysis() {
127+
const counted = PerformanceAnalysis.count();
128+
if (counted && !is_analysis) {
129+
PerformanceAnalysis.writePerformanceReport();
130+
console.log(
131+
'performance analysis finished, check the performanceRecord.txt file for details'
132+
);
133+
is_analysis = true;
134+
}
135+
}

packages/core/src/lib/utils/index.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,4 @@ export * from './module';
44
export * from './streams';
55
export * from './errors';
66
export * from './flattenElements';
7+
export * from './analyzer';

0 commit comments

Comments
 (0)