Skip to content

Commit

Permalink
Use inout/output feature from broccoli-plugin
Browse files Browse the repository at this point in the history 
Using fs-merger
1. Allow filter developers to reduce mergeTree or funneling entire array of trees to one tree and then pass it to filter
2. Allows input array of trees as input.

input/output :
1. Provides good ergonomics and avoids appending the this.outputPath

Test:
1. Added additional tests to make sure that Filter can take more than one tree
  • Loading branch information
sparshithNR committed Dec 6, 2019
1 parent 598264a commit 9e01bda
Show file tree
Hide file tree
Showing 8 changed files with 275 additions and 495 deletions.
96 changes: 56 additions & 40 deletions index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ const path = require('path');
const mkdirp = require('mkdirp');
const rimraf = require('rimraf');
const Plugin = require('broccoli-plugin');
const walkSync = require('walk-sync');
const mapSeries = require('promise-map-series');
const symlinkOrCopySync = require('symlink-or-copy').sync;
const debugGenerator = require('heimdalljs-logger');
Expand All @@ -18,6 +17,7 @@ const hashForDep = require('hash-for-dep');
const FSTree = require('fs-tree-diff');
const heimdall = require('heimdalljs');
const queue = require('async-promise-queue');
const FSMerger = require('fs-merger');

class ApplyPatchesSchema {
constructor() {
Expand Down Expand Up @@ -104,8 +104,15 @@ module.exports = class Filter extends Plugin {
return !!result;
}

constructor(inputTree, options) {
super([inputTree], {
constructor(inputTrees, options) {
// Parsed array of trees to have either broccoliNode or string which is expected by broccoli-plugin
let inputTreesParsed = [];
inputTrees = Array.isArray(inputTrees) ? inputTrees : [inputTrees];
inputTrees.forEach((tree) => {
inputTreesParsed.push(tree.root ? tree.root : tree);
});

super(inputTreesParsed, {
name: (options && options.name),
annotation: (options && options.annotation),
persistentOutput: true
Expand All @@ -121,6 +128,7 @@ module.exports = class Filter extends Plugin {
loggerName += ' > [' + annotation + ']';
}

this.fsMerger = new FSMerger(inputTrees);
/** @type {{debug(...s: any[]): void; info(...s: any[]): void}} */
// @ts-ignore
this._logger = debugGenerator(loggerName);
Expand Down Expand Up @@ -158,21 +166,17 @@ module.exports = class Filter extends Plugin {
}

async build() {
// @ts-ignore
let srcDir = this.inputPaths[0];
// @ts-ignore
let destDir = this.outputPath;

if (this.dependencyInvalidation && !this.dependencies) {
this.dependencies = this.processor.initialDependencies(srcDir);
this.dependencies = this.processor.initialDependencies(this.inputPaths);
}

if (this._needsReset) {
this.currentTree = new FSTree();
// @ts-ignore
let instrumentation = heimdall.start('reset');
if (this.dependencies) {
this.dependencies = this.processor.initialDependencies(srcDir);
this.dependencies = this.processor.initialDependencies(this.inputPaths);
}
// @ts-ignore
rimraf.sync(this.outputPath);
Expand All @@ -186,8 +190,8 @@ module.exports = class Filter extends Plugin {
let instrumentation = heimdall.start('derivePatches', DerivePatchesSchema);

let walkStart = process.hrtime();
let entries = walkSync.entries(srcDir);
let nextTree = FSTree.fromEntries(entries);
let entries = this.fsMerger.entries('');
let nextTree = FSTree.fromEntries(entries, { sortAndExpand: true });
let walkDuration = timeSince(walkStart);

let invalidationsStart = process.hrtime();
Expand Down Expand Up @@ -239,27 +243,35 @@ module.exports = class Filter extends Plugin {
let operation = patch[0];
let relativePath = patch[1];
let entry = patch[2];
let outputPath = destDir + '/' + (this.getDestFilePath(relativePath, entry) || relativePath);
let outputFilePath = outputPath;
let fileMeta = this.fsMerger.readFileMeta(relativePath, { basePath: entry.basePath });
let outputPath = relativePath;
if (fileMeta && fileMeta.getDestinationPath) {
outputPath = fileMeta.getDestinationPath(relativePath);
} else {
if (fileMeta && fileMeta.prefix) {
outputPath = fileMeta.prefix + '/' + outputPath;
}
}
outputPath = this.getDestFilePath(outputPath, entry) || outputPath;
let forceInvalidation = invalidated.includes(relativePath);

this._logger.debug('[operation:%s] %s', operation, relativePath);

switch (operation) {
case 'mkdir': {
instrumentation.mkdir++;
return fs.mkdirSync(outputPath);
return this.output.mkdirSync(outputPath, { recursive: true });
} case 'rmdir': {
instrumentation.rmdir++;
return fs.rmdirSync(outputPath);
return this.output.rmdirSync(outputPath);
} case 'unlink': {
instrumentation.unlink++;
return fs.unlinkSync(outputPath);
return fs.unlinkSync( this.outputPath + '/' + outputPath);
} case 'change': {
// wrap this in a function so it doesn't actually run yet, and can be throttled
let changeOperation = () => {
instrumentation.change++;
return this._handleFile(relativePath, srcDir, destDir, entry, outputFilePath, forceInvalidation, true, instrumentation);
return this._handleFile(relativePath, entry, outputPath, forceInvalidation, true, instrumentation);
};
if (this.async) {
pendingWork.push(changeOperation);
Expand All @@ -270,7 +282,7 @@ module.exports = class Filter extends Plugin {
// wrap this in a function so it doesn't actually run yet, and can be throttled
let createOperation = () => {
instrumentation.create++;
return this._handleFile(relativePath, srcDir, destDir, entry, outputFilePath, forceInvalidation, false, instrumentation);
return this._handleFile(relativePath, entry, outputPath, forceInvalidation, false, instrumentation);
};
if (this.async) {
pendingWork.push(createOperation);
Expand All @@ -292,27 +304,27 @@ module.exports = class Filter extends Plugin {
});
}

async _handleFile(relativePath, srcDir, destDir, entry, outputPath, forceInvalidation, isChange, stats) {
async _handleFile(relativePath, entry, outputPath, forceInvalidation, isChange, stats) {
stats.handleFile++;

let handleFileStart = process.hrtime();
let destDir = this.outputPath + '/' + outputPath;
try {
let result;
let srcPath = srcDir + '/' + relativePath;

if (this.canProcessFile(relativePath, entry)) {
stats.processed++;
if (this._outputLinks[outputPath] === true) {
delete this._outputLinks[outputPath];
fs.unlinkSync(outputPath);
fs.unlinkSync(destDir);
}
result = await this.processAndCacheFile(srcDir, destDir, entry, forceInvalidation, isChange, stats);
result = await this.processAndCacheFile(relativePath, entry, forceInvalidation, isChange, stats);
} else {
stats.linked++;
if (isChange) {
fs.unlinkSync(outputPath);
fs.unlinkSync(destDir);
}
result = symlinkOrCopySync(srcPath, outputPath);
result = symlinkOrCopySync(entry.fullPath, destDir);
this._outputLinks[outputPath] = true;
}
return result;
Expand Down Expand Up @@ -362,15 +374,15 @@ module.exports = class Filter extends Plugin {

isDirectory(relativePath, entry) {
// @ts-ignore
if (this.inputPaths === undefined) {
if (this.fsMerger === undefined) {
return false;
}

// @ts-ignore
let srcDir = this.inputPaths[0];
let path = srcDir + '/' + relativePath;

return (entry || fs.lstatSync(path)).isDirectory();
let fileMeta = this.fsMerger.readFileMeta(relativePath, { basePath: entry && entry.basePath });
// If fileMeta isn't present for a path indicates that path isn't present. We send false in this case.
if (!fileMeta) {
return false;
}
return (entry || fs.lstatSync(fileMeta.path)).isDirectory();
}

getDestFilePath(relativePath, entry) {
Expand All @@ -396,11 +408,11 @@ module.exports = class Filter extends Plugin {
return null;
}

async processAndCacheFile(srcDir, destDir, entry, forceInvalidation, isChange, instrumentation) {
async processAndCacheFile(srcDir, entry, forceInvalidation, isChange, instrumentation) {
let filter = this;
let relativePath = entry.relativePath;
try {
return await filter.processFile(srcDir, destDir, relativePath, forceInvalidation, isChange, instrumentation, entry);
return await filter.processFile(srcDir, relativePath, forceInvalidation, isChange, instrumentation, entry);
} catch (e) {
let error = e;
if (typeof e !== 'object') error = new Error('' + e);
Expand All @@ -410,17 +422,18 @@ module.exports = class Filter extends Plugin {
}
}

async processFile(srcDir, destDir, relativePath, forceInvalidation, isChange, instrumentation, entry) {
async processFile(srcDir, relativePath, forceInvalidation, isChange, instrumentation, entry) {
let filter = this;
let inputEncoding = this.inputEncoding;
let outputEncoding = this.outputEncoding;

if (inputEncoding === undefined) inputEncoding = 'utf8';
if (outputEncoding === undefined) outputEncoding = 'utf8';

let contents = fs.readFileSync(srcDir + '/' + relativePath, {
let contents = fs.readFileSync(entry.fullPath, {
encoding: inputEncoding
});
let fileMeta = this.fsMerger.readFileMeta(srcDir, { basePath: entry.basePath });

instrumentation.processString++;
let processStringStart = process.hrtime();
Expand All @@ -434,10 +447,13 @@ module.exports = class Filter extends Plugin {
relativePath + '") is null');
}

outputPath = destDir + '/' + outputPath;
if (fileMeta && fileMeta.getDestinationPath) {
outputPath = fileMeta.getDestinationPath(outputPath);
}
outputPath = path.join(fileMeta.prefix || '', outputPath);

if (isChange) {
let isSame = fs.readFileSync(outputPath, 'UTF-8') === outputString;
let isSame = this.output.readFileSync(outputPath, 'UTF-8') === outputString;
if (isSame) {
this._logger.debug('[change:%s] but was the same, skipping', relativePath, isSame);
return;
Expand All @@ -447,14 +463,14 @@ module.exports = class Filter extends Plugin {
}

try {
fs.writeFileSync(outputPath, outputString, {
this.output.writeFileSync(outputPath, outputString, {
encoding: outputEncoding
});

} catch (e) {
if (e !== null && typeof e === 'object' && e.code === 'ENOENT') {
mkdirp.sync(path.dirname(outputPath));
fs.writeFileSync(outputPath, outputString, {
this.output.mkdirSync(path.dirname(outputPath), { recursive: true });
this.output.writeFileSync(outputPath, outputString, {
encoding: outputEncoding
});
} else {
Expand Down
Loading

0 comments on commit 9e01bda

Please sign in to comment.