Skip to content

Commit

Permalink
Allow mutiple trees as input
Browse files Browse the repository at this point in the history 
pass customFS in dependencies as we want to pass encapsulation of this.input
  • Loading branch information
sparshithNR committed Dec 19, 2019
1 parent 7c754d0 commit 5bac413
Show file tree
Hide file tree
Showing 8 changed files with 156 additions and 76 deletions.
10 changes: 5 additions & 5 deletions index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@

const path = require('path');
const Plugin = require('broccoli-plugin');
const walkSync = require('walk-sync');
const mapSeries = require('promise-map-series');
const debugGenerator = require('heimdalljs-logger');
const md5Hex = require('./lib/md5-hex');
Expand Down Expand Up @@ -101,7 +100,8 @@ module.exports = class Filter extends Plugin {
}

constructor(inputTree, options) {
super([inputTree], {
inputTree = Array.isArray(inputTree) ? inputTree : [inputTree];
super(inputTree, {
name: (options && options.name),
annotation: (options && options.annotation),
persistentOutput: true
Expand Down Expand Up @@ -160,15 +160,15 @@ module.exports = class Filter extends Plugin {
let destDir = this.outputPath;

if (this.dependencyInvalidation && !this.dependencies) {
this.dependencies = this.processor.initialDependencies(srcDir);
this.dependencies = this.processor.initialDependencies(this.inputPaths, { fs: this.input });
}

if (this._needsReset) {
this.currentTree = new FSTree();
// @ts-ignore
let instrumentation = heimdall.start('reset');
if (this.dependencies) {
this.dependencies = this.processor.initialDependencies(srcDir);
this.dependencies = this.processor.initialDependencies(this.inputPaths, { fs: this.input });
}
// @ts-ignore
this.output.rmdirSync('./', { recursive: true });
Expand All @@ -182,7 +182,7 @@ module.exports = class Filter extends Plugin {
let instrumentation = heimdall.start('derivePatches', DerivePatchesSchema);

let walkStart = process.hrtime();
let entries = walkSync.entries(srcDir);
let entries = this.input.entries('./');
let nextTree = FSTree.fromEntries(entries);
let walkDuration = timeSince(walkStart);

Expand Down
85 changes: 55 additions & 30 deletions lib/dependencies.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,21 @@ const FSHashTree = fsHashDiff.FSHashTree;
module.exports = class Dependencies {
/**
* Creates an instance of Dependencies.
* @param rootDir {string} The root directory containing the files that
* @param inputPaths {Array<string>} The root directories containing the files that
* have dependencies. Relative paths are resolved against this directory.
* @param options { {[key:string]: any} } options is used to pass the custom fs opertations implementations
*/
constructor(rootDir) {
constructor(inputPaths, options = {}) {
/**
* Tracks whether new dependencies can be added.
**/
this.sealed = false;
/**
* The root directory containing the files that have dependencies. Relative
* The root directories containing the files that have dependencies. Relative
* paths are resolved against this directory.
* @type {string}
* @type {Array<string>}
*/
this.rootDir = path.normalize(rootDir);
this.rootDirs = Array.isArray(inputPaths) ? inputPaths : [inputPaths];
/**
* Tracks dependencies on a per file basis.
* The key is a relative path, values are absolute paths.
Expand All @@ -52,6 +53,13 @@ module.exports = class Dependencies {
* @type Map<string, Array<string>>;
*/
this.dependentsMap = new Map();
/**
* Custom fs object can be passed to the custructor.
* This helps us to pass the this.input of the broccoli-plugin
* to keep the encapsulations.
* @type {typeof fs}
*/
this.fs = options.fs || fs;
}

/**
Expand All @@ -67,9 +75,13 @@ module.exports = class Dependencies {
// Build a unified set of dependencies for the entire tree
/** @type {string} */
let depRoot;
if (deps[i].startsWith(this.rootDir + path.sep)) {
depRoot = this.rootDir;
} else {
for (let index = 0; index < this.rootDirs.length; index++) {
if (deps[i].startsWith(path.normalize(this.rootDirs[index]) + path.sep)) {
depRoot = this.rootDirs[index];
break;
}
}
if (!depRoot) {
depRoot = path.parse(deps[i]).root;
}
let depsForRoot = this._getDepsForRoot(depRoot);
Expand Down Expand Up @@ -139,11 +151,10 @@ module.exports = class Dependencies {
}
/** @type {Array<string>} */
let absoluteDeps = [];
let fileDir = path.dirname(filePath);
for (let i = 0; i < dependencies.length; i++) {
let depPath = path.normalize(dependencies[i]);
if (!path.isAbsolute(depPath)) {
depPath = path.resolve(this.rootDir, fileDir, depPath);
throw Error(`setDependencies now accepts only absolute path for dependecies path list`);
}
absoluteDeps.push(depPath);
}
Expand All @@ -162,7 +173,7 @@ module.exports = class Dependencies {
*/
copyWithout(files) {
files = files.map(f => path.normalize(f));
let newDeps = new Dependencies(this.rootDir);
let newDeps = new Dependencies(this.rootDirs, { fs: this.fs });
for (let file of this.dependencyMap.keys()) {
if (!files.includes(file)) {
newDeps.setDependencies(file, this.dependencyMap.get(file));
Expand Down Expand Up @@ -195,10 +206,10 @@ module.exports = class Dependencies {
let dependencies = this.allDependencies.get(fsRoot);
/** @type {FSTree<Entry> | FSHashTree} */
let fsTree;
if (fsRoot === this.rootDir) {
fsTree = getHashTree(fsRoot, dependencies);
if (this.rootDirs.indexOf(fsRoot) > -1) {
fsTree = getHashTree(fsRoot, dependencies, this.fs);
} else {
fsTree = getStatTree(fsRoot, dependencies);
fsTree = getStatTree(fsRoot, dependencies, this.fs);
}
fsTrees.set(fsRoot, fsTree);
}
Expand Down Expand Up @@ -237,7 +248,7 @@ module.exports = class Dependencies {
*
* This object is serializable so it can be put into the persistent cache and
* used to invalidate files during the next build in a new process.
* @return {{rootDir: string, dependencies: {[k: string]: string[]}, fsTrees: Array<{fsRoot: string, entries: Array<{relativePath: string} & ({type: 'stat', size: number, mtime: number, mode: number} | {type: 'hash', hash: string})>}>}}
* @return {{rootDirs: Array<string>, dependencies: {[k: string]: string[]}, fsTrees: Array<{fsRoot: string, entries: Array<{relativePath: string} & ({type: 'stat', size: number, mtime: number, mode: number} | {type: 'hash', hash: string})>}>}}
*/
serialize() {
/** @type {{[k: string]: string[]}} */
Expand Down Expand Up @@ -274,7 +285,7 @@ module.exports = class Dependencies {
});
}
let serialized = {
rootDir: this.rootDir,
rootDirs: this.rootDirs,
dependencies,
fsTrees
};
Expand All @@ -285,21 +296,29 @@ module.exports = class Dependencies {
* Deserialize from JSON data returned from the `serialize` method.
*
* @param dependencyData {ReturnType<Dependencies['serialize']>}
* @param [newRootDir] {string | undefined}
* @param [newRootDirs] {Array<string> | undefined}
* @return {Dependencies};
*/
static deserialize(dependencyData, newRootDir) {
let oldRootDir = dependencyData.rootDir;
newRootDir = path.normalize(newRootDir || oldRootDir);
let dependencies = new Dependencies(newRootDir);
static deserialize(dependencyData, newRootDirs, customFS = fs) {
let oldRootDirs = dependencyData.rootDirs;
oldRootDirs.sort();
newRootDirs = newRootDirs || oldRootDirs;
let dependencies = new Dependencies(newRootDirs, { fs: customFS });
newRootDirs.sort();
let files = Object.keys(dependencyData.dependencies);
for (let file of files) {
let deps = dependencyData.dependencies[file];
if (newRootDir) {
if (newRootDirs) {
for (let i = 0; i < deps.length; i++) {
let dep = deps[i];
if (dep.startsWith(oldRootDir+path.sep)) {
deps[i] = dep.replace(oldRootDir, newRootDir);
for (let index = 0; index < oldRootDirs.length; index++) {
if (dep.startsWith(path.normalize(oldRootDirs[index]) + path.sep)) {
let newPath = dep.replace(oldRootDirs[index], newRootDirs[index]);
if (fs.existsSync(newPath)) {
deps[i] = newPath;
break;
}
}
}
}
}
Expand All @@ -321,10 +340,16 @@ module.exports = class Dependencies {
let fsTree;
/** @type {string} */
let treeRoot;
if (fsTreeData.fsRoot === oldRootDir) {
treeRoot = newRootDir;
fsTree = FSHashTree.fromHashEntries(entries, { sortAndExpand: true });
} else {
for (let index = 0; index < oldRootDirs.length; index++) {
const oldRootDir = oldRootDirs[index];
if (fsTreeData.fsRoot === oldRootDir) {
// need to check this swapping logic. newRootDirs may not be in same sequence as it was with oldRootDirs.
treeRoot = newRootDirs[index];
fsTree = FSHashTree.fromHashEntries(entries, { sortAndExpand: true });
break;
}
}
if (!(treeRoot || fsTree)) {
treeRoot = fsTreeData.fsRoot;
fsTree = FSTree.fromEntries(entries, { sortAndExpand: true });
}
Expand All @@ -344,7 +369,7 @@ module.exports = class Dependencies {
* @param dependencies {Set<string>}
* @return {FSHashTree}
*/
function getHashTree(fsRoot, dependencies) {
function getHashTree(fsRoot, dependencies, fs) {
/** @type {Array<HashEntry>} */
let entries = [];
for (let dependency of dependencies) {
Expand All @@ -369,7 +394,7 @@ function getHashTree(fsRoot, dependencies) {
* @param fsRoot {string}
* @param dependencies {Set<string>}
*/
function getStatTree(fsRoot, dependencies) {
function getStatTree(fsRoot, dependencies, fs) {
/** @type {Array<Entry>} */
let entries = [];
for (let dependency of dependencies) {
Expand Down
7 changes: 4 additions & 3 deletions lib/processor.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,12 @@ module.exports = class Processor {

/**
* Create the initial dependencies.
* @param srcDir {string}
* @param inputPaths {string[]}
* @param options { {[key:string]: any} } options is used to pass the custom fs opertations implementations
* @returns {ReturnType<typeof defaultProcessor['initialDependencies']>}
*/
initialDependencies(srcDir) {
return this.processor.initialDependencies(srcDir);
initialDependencies(inputPaths, options) {
return this.processor.initialDependencies(inputPaths, options);
}

/**
Expand Down
4 changes: 2 additions & 2 deletions lib/strategies/default.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,10 @@ module.exports = {
* By default initial dependencies are empty.
* @returns {Dependencies}
*/
initialDependencies(srcDir) {
initialDependencies(inputPaths, options) {
// Dependencies start out empty and sealed as if they came from
// the previous build iteration.
return (new Dependencies(srcDir)).seal().captureDependencyState();
return (new Dependencies(inputPaths, options)).seal().captureDependencyState();
},

/**
Expand Down
6 changes: 3 additions & 3 deletions lib/strategies/persistent.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,17 +85,17 @@ module.exports = {
* By default initial dependencies are empty.
* @returns {Dependencies}
*/
initialDependencies(srcDir) {
initialDependencies(inputPaths, options = {}) {
let result = this._syncCache.get('__dependencies__');
let dependencies;
if (result.isCached) {
/** @type {ReturnType<Dependencies['serialize']>} */
let data = JSON.parse(result.value);
dependencies = Dependencies.deserialize(data, srcDir);
dependencies = Dependencies.deserialize(data, inputPaths, options.fs);
} else {
// Dependencies start out empty; they are sealed as if they came from
// the previous build iteration.
dependencies = new Dependencies(srcDir);
dependencies = new Dependencies(inputPaths, options);
dependencies.seal().captureDependencyState();
}
return dependencies;
Expand Down
3 changes: 1 addition & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,7 @@
"promise-map-series": "^0.2.1",
"rimraf": "^3.0.0",
"symlink-or-copy": "^1.0.1",
"sync-disk-cache": "^2.0.0",
"walk-sync": "^2.0.2"
"sync-disk-cache": "^2.0.0"
},
"devDependencies": {
"broccoli-test-helper": "^2.0.0",
Expand Down
Loading

0 comments on commit 5bac413

Please sign in to comment.