Skip to content

Commit 2e54304

Browse files
docd27docd27
committed
Fix parallel builds, sourcemaps
1 parent 3654743 commit 2e54304

File tree

3 files changed

+150
-102
lines changed

3 files changed

+150
-102
lines changed

‎src/index.js‎

Lines changed: 23 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import {createFilter} from '@rollup/pluginutils';
2-
import MagicString from 'magic-string';
32
import { glslProcessSource } from './lib/glslProcess.js';
3+
import * as fsSync from 'fs';
44

55
/**
66
* @typedef {import('./lib/glslProcess').GLSLStageName} GLSLStageName
@@ -18,7 +18,9 @@ const stageDefs = {
1818

1919
const extsInclude = Object.values(stageDefs).flatMap(
2020
(exts) => exts.map((ext) => `**/*${ext}`));
21-
const stageRegexes = new Map(
21+
22+
/** @type {[GLSLStageName, RegExp][]} */
23+
const stageRegexes = (
2224
/** @type {[GLSLStageName, string[]][]} */(Object.entries(stageDefs))
2325
.map(([st, exts]) => [st,
2426
new RegExp(`(?:${exts.map(ext => ext.replace('.', '\\.')).join('|')})$`, 'i')
@@ -35,8 +37,6 @@ function generateCode(source) {
3537
* File extensions within rollup to include.
3638
* @property {PathFilter} [exclude]
3739
* File extensions within rollup to exclude.
38-
* @property {boolean} [sourceMap]
39-
* Emit source maps
4040
* @typedef {GLSLPluginGlobalOptions & Partial<import('./lib/glslProcess').GLSLToolOptions>} GLSLPluginOptions
4141
*/
4242
/**
@@ -47,7 +47,6 @@ export default function glslOptimize(userOptions = {}) {
4747
/** @type {GLSLPluginOptions} */
4848
const options = {
4949
include: extsInclude,
50-
sourceMap: true,
5150
...userOptions,
5251
};
5352

@@ -56,42 +55,37 @@ export default function glslOptimize(userOptions = {}) {
5655
return {
5756
name: 'glsl-optimize',
5857

59-
async transform(code, id) {
58+
async load(id) {
6059
if (!id || !filter(id)) return;
6160

62-
/** @type {GLSLStageName} */
63-
let stage;
64-
for (const [checkStage, regex] of stageRegexes) {
65-
if (id.match(regex)) {
66-
stage = checkStage;
67-
break;
68-
}
61+
/*
62+
We use a load hook instead of transform because we want sourcemaps
63+
to reflect the optimized shader source.
64+
*/
65+
if (!fsSync.existsSync(id)) return;
66+
let source;
67+
try {
68+
source = fsSync.readFileSync(id, {encoding: 'utf8'});
69+
} catch (err) {
70+
this.warn(`Failed to load file '${id}' : ${err.message}`);
71+
return;
6972
}
73+
74+
/** @type {GLSLStageName} */
75+
const stage = stageRegexes.find(([, regex]) => id.match(regex))?.[0];
7076
if (!stage) {
7177
this.error({ message: `File '${id}' : extension did not match a shader stage.` });
7278
return;
7379
}
80+
7481
try {
75-
code = await glslProcessSource(id, code, stage, options);
82+
const result = await glslProcessSource(id, source, stage, options);
83+
result.code = generateCode(result.code);
84+
return result;
7685
} catch (err) {
7786
this.error({ message: `Error processing GLSL source:\n${err.message}` });
7887
return;
7988
}
80-
81-
code = generateCode(code);
82-
83-
if (options.sourceMap !== false) {
84-
const magicString = new MagicString(code);
85-
return {
86-
code: magicString.toString(),
87-
map: magicString.generateMap({hires: true}),
88-
};
89-
} else {
90-
return {
91-
code,
92-
map: {mappings: ''},
93-
};
94-
}
9589
},
9690
};
9791
}

‎src/lib/glslProcess.js‎

Lines changed: 50 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,19 @@
11
import {EOL} from 'os';
22
import * as path from 'path';
33
import * as fsSync from 'fs';
4-
import {insertExtensionPreamble, fixupDirectives} from './preamble.js';
4+
import {insertExtensionPreamble, fixupDirectives, insertPreamble} from './preamble.js';
55
import {argQuote, configureTools, getCachePath, launchTool, waitForToolBuffered} from './tools.js';
66
import {checkMakeFolder, rmDir} from './download.js';
7-
import { compressShader } from './minify.js';
7+
import {compressShader} from './minify.js';
8+
import * as crypto from 'crypto';
9+
import MagicString from 'magic-string';
810

911
/**
1012
* @typedef {'vert'|'tesc'|'tese'|'geom'|'frag'|'comp'} GLSLStageName
1113
*
1214
* @typedef {Object} GLSLToolSharedOptions
15+
* @property {boolean} sourceMap
16+
* Emit source maps
1317
* @property {boolean} compress
1418
* Strip whitespace
1519
* @property {boolean} optimize
@@ -127,18 +131,32 @@ async function glslRunCross(name, workingDir, stageName, inputFile, input, emitL
127131
]);
128132
}
129133

134+
/**
135+
* Generate unique build path
136+
* @param {string} id
137+
* @return {string}
138+
*/
139+
function getBuildDir(id) {
140+
const sanitizeID = path.basename(id).replace(/([^a-z0-9]+)/gi, '-').toLowerCase();
141+
const uniqID = ((Date.now()>>>0) + crypto.randomBytes(4).readUInt32LE())>>>0; // +ve 4 byte unique ID
142+
const uniqIDHex = uniqID.toString(16).padStart(8, '0'); // 8 char random hex
143+
return path.join(getCachePath(), 'glslBuild', `${sanitizeID}-${uniqIDHex}`);
144+
}
145+
130146
/**
131147
* @internal
132148
* @param {string} id File path
133149
* @param {string} source Source code
134150
* @param {GLSLStageName} stageName
135151
* @param {Partial<GLSLToolOptions>} [glslOptions]
136152
* @param {(message: string) => void} [errorLog]
153+
* @return {Promise<import('rollup').SourceDescription>}
137154
*/
138155
export async function glslProcessSource(id, source, stageName, glslOptions = {}, errorLog = console.error) {
139156

140157
/** @type {GLSLToolOptions} */
141158
const options = {
159+
sourceMap: true,
142160
compress: true,
143161
optimize: true,
144162
emitLineDirectives: false,
@@ -157,7 +175,7 @@ export async function glslProcessSource(id, source, stageName, glslOptions = {},
157175

158176
let tempBuildDir;
159177
if (options.optimize) {
160-
tempBuildDir = path.join(getCachePath(), 'glslBuild');
178+
tempBuildDir = getBuildDir(id);
161179
rmDir(tempBuildDir);
162180
checkMakeFolder(tempBuildDir);
163181
}
@@ -209,7 +227,7 @@ export async function glslProcessSource(id, source, stageName, glslOptions = {},
209227
...options.extraValidatorParams,
210228
];
211229

212-
let outputGLSL;
230+
let processedGLSL;
213231

214232
if (options.optimize) {
215233
const outputBuild = await glslRunValidator('Build spirv', targetDir, stageName,
@@ -233,11 +251,11 @@ export async function glslProcessSource(id, source, stageName, glslOptions = {},
233251
// '--print-all', // Print spirv for debugging
234252
], options.extraOptimizerParams);
235253
if (!fsSync.existsSync(optimizedFileAbs)) {
236-
throw new Error(`Optimize spirv failed: no output file`);
254+
throw new Error(`Optimize spirv failed: no output file (${optimizedFileAbs})`);
237255
}
238256
}
239257

240-
outputGLSL = await glslRunCross('Build spirv to GLSL', targetDir, stageName,
258+
processedGLSL = await glslRunCross('Build spirv to GLSL', targetDir, stageName,
241259
options.optimizerDebugSkipOptimizer ? outputFileAbs : optimizedFileAbs, undefined, options.emitLineDirectives, [
242260
'--es', // WebGL is always ES
243261
'--version', `${targetGlslVersion}`,
@@ -252,22 +270,42 @@ export async function glslProcessSource(id, source, stageName, glslOptions = {},
252270

253271

254272
} else {
255-
outputGLSL = await glslRunValidator('Preprocessing', targetDir, stageName, code, [
273+
processedGLSL = await glslRunValidator('Preprocessing', targetDir, stageName, code, [
256274
'-E', // print pre-processed GLSL
257275
], extraValidatorParams);
258276
const outputValidated = await glslRunValidator('Validation', targetDir, stageName,
259-
outputGLSL, [], extraValidatorParams);
277+
processedGLSL, [], extraValidatorParams);
260278
}
261279

262-
outputGLSL = fixupDirectives(outputGLSL,
280+
processedGLSL = fixupDirectives(processedGLSL,
263281
options.emitLineDirectives && !options.suppressLineExtensionDirective,
264282
didInsertion && (!options.optimize || options.emitLineDirectives),
265283
options.optimize, !options.emitLineDirectives, undefined);
266284

267-
if (options.compress) {
268-
outputGLSL = compressShader(outputGLSL);
285+
286+
const outputCode = options.compress ? compressShader(processedGLSL) : processedGLSL;
287+
288+
/** @type {import('rollup').LoadResult} */
289+
const result = {
290+
code: outputCode,
291+
map: {mappings: ''},
292+
};
293+
294+
if (options.sourceMap) {
295+
const sourceMapSource = insertPreamble(processedGLSL,
296+
'/*\n' +
297+
`* Preprocessed${options.optimize?' + Optimized':''} from '${targetID}'\n` +
298+
(options.compress ? '* [Embedded string is compressed]\n':'') +
299+
'*/'
300+
).code;
301+
const magicString = new MagicString(sourceMapSource);
302+
result.map = magicString.generateMap({
303+
source: id,
304+
includeContent: true,
305+
hires: true,
306+
});
269307
}
270308

271-
return outputGLSL;
309+
return result;
272310

273311
}

‎src/lib/preamble.js‎

Lines changed: 77 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -29,67 +29,11 @@ export function insertExtensionPreamble(code, filePath, versionReplacer = (v) =>
2929
}
3030
})()];
3131

32-
/**
33-
* @param {number} fixupLineNo
34-
* @return {import('./parse.js').ParserToken}
35-
* Preamble with correct #line directive fixup
36-
*/
37-
const preambleToken = (fixupLineNo) => ({col: 0, line: fixupLineNo, type: TOK.Directive, value: '', text:
38-
`#extension ${GLSL_INCLUDE_EXT} : require\n${
39-
extraPreamble ? `${extraPreamble}\n` : ''}#line ${fixupLineNo} "${filePath}"\n`});
40-
41-
const versionToken = function* (token) {
42-
yield {type: TOK.Version, Version: versionReplacer(undefined), col: token.col, line: token.line,
43-
text: `#version ${versionReplacer(undefined)}`, value: ''};
44-
yield {type: TOK.EOL, col: token.col, line: token.line, text: '\n', value: '\n'};
45-
};
46-
47-
return {code: [...(/** @return {Generator<import('./parse.js').ParserToken>} */ function* () {
48-
let insertNext = false, acceptVersion = true, foundVersion = false, didInsertion = false;
49-
for (const token of tokens) {
50-
if (insertNext) {
51-
insertNext = false;
52-
yield preambleToken(token.line);
53-
}
54-
switch (token.type) {
55-
case TOK.Comment: break;
56-
case TOK.EOF:
57-
if (!didInsertion) {
58-
didInsertion = true;
59-
// Needs a new line
60-
if (acceptVersion) yield* versionToken(token);
61-
yield {type: TOK.EOL, col: token.col, line: token.line, text: '\n', value: '\n'};
62-
yield preambleToken(token.line + 1);
63-
}
64-
break;
65-
case TOK.EOL:
66-
if (acceptVersion) yield* versionToken(token);
67-
acceptVersion = false;
68-
if (!didInsertion) {
69-
didInsertion = true; insertNext = true;
70-
}
71-
break;
72-
case TOK.Version:
73-
if (acceptVersion) {
74-
acceptVersion = false; foundVersion = true;
75-
const newVersion = versionReplacer(token.Version);
76-
token.Version = newVersion;
77-
token.text = `#version ${newVersion}`;
78-
} else {
79-
throw new Error(formatParseError(`Parse error: #version directive must be on first line`, token));
80-
}
81-
break;
82-
default:
83-
if (acceptVersion) yield* versionToken(token);
84-
acceptVersion = false;
85-
}
86-
yield token;
87-
}
88-
if (!foundVersion) {
89-
console.warn(`Warning: #version directive missing`);
90-
return code;
91-
}
92-
})()].map((tok) => tok.text).join(''), didInsertion: true};
32+
return insertPreambleTokens(tokens,
33+
(fixupLineNo) => ({col: 0, line: fixupLineNo, type: TOK.Directive, value: '', text:
34+
`#extension ${GLSL_INCLUDE_EXT} : require${
35+
extraPreamble ? `\n${extraPreamble}` : ''}\n#line ${fixupLineNo} "${filePath}"\n`}),
36+
versionReplacer);
9337
}
9438

9539

@@ -155,3 +99,75 @@ export function fixupDirectives(code, preserve = false, required = true, searchL
15599
}
156100
})()].map((tok) => tok.text).join('');
157101
}
102+
103+
/**
104+
* @internal
105+
* @param {Iterable<import('./parse.js').ParserToken>} tokens
106+
* @param {(fixupLineNo: number) => import('./parse.js').ParserToken} preambleToken
107+
* @param {(version: string) => string} [versionReplacer]
108+
* @return {{code: string, didInsertion: boolean, foundVersionString?: string}}
109+
*/
110+
function insertPreambleTokens(tokens, preambleToken, versionReplacer = (v) => v) {
111+
const versionToken = function* (token) {
112+
yield {type: TOK.Version, Version: versionReplacer(undefined), col: token.col, line: token.line,
113+
text: `#version ${versionReplacer(undefined)}`, value: ''};
114+
yield {type: TOK.EOL, col: token.col, line: token.line, text: '\n', value: '\n'};
115+
};
116+
117+
return {code: [...(/** @return {Generator<import('./parse.js').ParserToken>} */ function* () {
118+
let insertNext = false, acceptVersion = true, foundVersion = false, didInsertion = false;
119+
for (const token of tokens) {
120+
if (insertNext) {
121+
insertNext = false;
122+
yield preambleToken(token.line);
123+
}
124+
switch (token.type) {
125+
case TOK.Comment: break;
126+
case TOK.EOF:
127+
if (!didInsertion) {
128+
didInsertion = true;
129+
// Needs a new line
130+
if (acceptVersion) yield* versionToken(token);
131+
yield {type: TOK.EOL, col: token.col, line: token.line, text: '\n', value: '\n'};
132+
yield preambleToken(token.line + 1);
133+
}
134+
break;
135+
case TOK.EOL:
136+
if (acceptVersion) yield* versionToken(token);
137+
acceptVersion = false;
138+
if (!didInsertion) {
139+
didInsertion = true; insertNext = true;
140+
}
141+
break;
142+
case TOK.Version:
143+
if (acceptVersion) {
144+
acceptVersion = false; foundVersion = true;
145+
const newVersion = versionReplacer(token.Version);
146+
token.Version = newVersion;
147+
token.text = `#version ${newVersion}`;
148+
} else {
149+
throw new Error(formatParseError(`Parse error: #version directive must be on first line`, token));
150+
}
151+
break;
152+
default:
153+
if (acceptVersion) yield* versionToken(token);
154+
acceptVersion = false;
155+
}
156+
yield token;
157+
}
158+
if (!foundVersion) {
159+
console.warn(`Warning: #version directive missing`);
160+
}
161+
})()].map((tok) => tok.text).join(''), didInsertion: true};
162+
}
163+
164+
/**
165+
* @internal
166+
* @param {string} code
167+
* @param {string} preamble
168+
*/
169+
export function insertPreamble(code, preamble) {
170+
return insertPreambleTokens(simpleParse(code),
171+
(fixupLineNo) => ({col: 0, line: fixupLineNo, type: TOK.Comment, value: '', text:
172+
`${preamble}\n`}));
173+
}

0 commit comments

Comments
 (0)