Skip to content

Commit

Permalink
Merge pull request #70 from robcresswell/feat/support-ignoring-paths
Browse files Browse the repository at this point in the history
feat: Support ignoring paths
  • Loading branch information
robcresswell authored Aug 4, 2019
2 parents a0b8f9d + 49bcabe commit cf7c78b
Show file tree
Hide file tree
Showing 14 changed files with 191 additions and 150 deletions.
4 changes: 4 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -41,5 +41,9 @@
"tar-fs": "^2.0.0",
"ts-jest": "^24.0.2",
"typescript": "^3.5.3"
},
"dependencies": {
"@types/minimatch": "^3.0.3",
"minimatch": "^3.0.4"
}
}
12 changes: 8 additions & 4 deletions src/cli.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { join } from 'path';
import { walkAndRender } from './walk';
import { help } from './help';
import { getValidConfig } from './validate';
Expand All @@ -19,8 +20,8 @@ export async function cli(
}

try {
const { resolvedDest, archiveUrl } = await getValidConfig(...args);
const templateVariables = await downloadRepo(archiveUrl, resolvedDest);
const config = await getValidConfig(...args);
const templateVariables = await downloadRepo(config);

if (templateVariables.size > 0) {
const variables: { [key: string]: string } = {};
Expand All @@ -35,12 +36,15 @@ export async function cli(

debug(`Variables: ${JSON.stringify(variables, null, 2)}`);

await walkAndRender(resolvedDest, variables);
const resolvedGlobs = config.ignoreGlobs.map((glob) =>
join(config.resolvedDest, glob),
);
await walkAndRender(config.resolvedDest, variables, resolvedGlobs);
}

return {
code: 0,
message: `Successfully downloaded to ${resolvedDest}`,
message: `Successfully downloaded to ${config.resolvedDest}`,
};
} catch ({ code, message }) {
return { code, message };
Expand Down
55 changes: 33 additions & 22 deletions src/download-repo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ import { createGunzip } from 'zlib';
import { extract, Headers } from 'tar-fs';
import { ReadStream } from 'fs';
import { parse } from 'mustache';
import minimatch from 'minimatch';
import { Config } from './types';

function map(header: Headers) {
// eslint-disable-next-line no-param-reassign
Expand All @@ -13,37 +15,41 @@ function map(header: Headers) {
return header;
}

function getMapStream(templateVariables: Set<string>) {
return (fileStream: ReadStream, { type }: Headers) => {
if (type === 'file') {
const templateChunks: string[] = [];

fileStream.on('data', (chunk) => templateChunks.push(chunk.toString()));
fileStream.on('end', () => {
const template: string[] = parse(templateChunks.join(''));
template
.filter((entry) => entry[0] === 'name')
.map((entry) => entry[1])
.forEach((entry) => templateVariables.add(entry));
});

function getMapStream(templateVariables: Set<string>, globsToIgnore: string[]) {
return (fileStream: ReadStream, { type, name }: Headers) => {
if (
type !== 'file' ||
globsToIgnore.some((glob) => minimatch(name, glob))
) {
return fileStream;
}

const templateChunks: string[] = [];

fileStream.on('data', (chunk) => templateChunks.push(chunk.toString()));
fileStream.on('end', () => {
const template: string[] = parse(templateChunks.join(''));
template
.filter((entry) => entry[0] === 'name')
.map((entry) => entry[1])
.forEach((entry) => templateVariables.add(entry));
});

return fileStream;
};
}

export async function downloadRepo(
archiveUrl: string,
dest: string,
): Promise<Set<string>> {
export async function downloadRepo({
archiveUrl,
resolvedDest,
ignoreGlobs,
}: Config): Promise<Set<string>> {
return new Promise((resolve, reject) => {
// TODO: Clean up this side-effect nightmare
const templateVariables: Set<string> = new Set();

// eslint-disable-next-line consistent-return
get(archiveUrl, (response) => {
get(archiveUrl, async (response) => {
if (!response.statusCode || response.statusCode >= 400) {
return reject(new Error(response.statusMessage));
}
Expand All @@ -59,13 +65,18 @@ export async function downloadRepo(
);
}

downloadRepo(redirectUrl, dest).then(resolve, reject);
downloadRepo({
archiveUrl: redirectUrl,
resolvedDest,
ignoreGlobs,
}).then(resolve, reject);
} else {
const mapStream = getMapStream(templateVariables);
const mapStream = getMapStream(templateVariables, ignoreGlobs);

response
.pipe(createGunzip())
.pipe(
extract(dest, {
extract(resolvedDest, {
map,
mapStream,
}),
Expand Down
35 changes: 35 additions & 0 deletions src/get-globs-to-ignore.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
import { get } from 'https';
import { debug } from './log';

export async function getGlobsToIgnore(
ignoreFileUrl: string,
): Promise<string[]> {
let globs: string[] = [];

return new Promise((resolve) => {
try {
get(ignoreFileUrl, (response) => {
if (response.statusCode !== 200) {
debug(`No .demsignore file found at ${ignoreFileUrl}`);
resolve([]);
}

const globChunks: string[] = [];

response.on('data', (chunk) => {
globChunks.push(chunk);
});

response.on('end', () => {
globs = globChunks
.join('')
.split('\n')
.filter(Boolean);
resolve(globs);
});
});
} catch {
debug(`Error retrieving .demsignore file at ${ignoreFileUrl}`);
}
});
}
22 changes: 0 additions & 22 deletions src/get-scm-info.ts

This file was deleted.

2 changes: 0 additions & 2 deletions src/prompt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ import { createInterface } from 'readline';
/**
* Prompts for user input, displaying the given `question` string, and returns
* the users input a a string
*
* @param question
*/
export async function prompt(question: string): Promise<string> {
const rl = createInterface({
Expand Down
1 change: 1 addition & 0 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ export type SCMType = 'github' | 'gitlab' | 'bitbucket';
export interface Config {
resolvedDest: string;
archiveUrl: string;
ignoreGlobs: string[];
}

export interface CommitSHAMap {
Expand Down
45 changes: 40 additions & 5 deletions src/validate.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,20 @@
import { resolve } from 'path';
import { existsSync } from 'fs';
import { MissingSourceArgError, DestExistsError } from './errors';
import { Config, CommitSHAMap } from './types';
import {
MissingSourceArgError,
DestExistsError,
InvalidSourceError,
} from './errors';
import { Config, CommitSHAMap, SCMType } from './types';
import { pExec } from './exec-promise';
import { debug } from './log';
import { getScmInfo } from './get-scm-info';
import { getGlobsToIgnore } from './get-globs-to-ignore';

const repoUrlMap = {
github: (repo: string) => `https://github.com/${repo}`,
gitlab: (repo: string) => `https://gitlab.com/${repo}`,
bitbucket: (repo: string) => `https://bitbucket.org/${repo}`,
};

const scmArchiveUrls = {
gitlab: (url: string, sha: string) =>
Expand All @@ -13,6 +23,15 @@ const scmArchiveUrls = {
github: (url: string, sha: string) => `${url}/archive/${sha}.tar.gz`,
};

const ignoreFileUrls = {
gitlab: (repo: string, sha: string) =>
`https://gitlab.com/${repo}/raw/${sha}/.demsignore`,
bitbucket: (repo: string, sha: string) =>
`https://bitbucket.org/${repo}/raw/${sha}/.demsignore`,
github: (repo: string, sha: string) =>
`https://raw.githubusercontent.com/${repo}/${sha}/.demsignore`,
};

/**
* Run several checks to see if the user input appears valid. If valid,
* transform it into a stricter structure for the rest of the program.
Expand All @@ -34,7 +53,17 @@ export async function getValidConfig(
}

debug('Checking source is valid');
const { url, type } = getScmInfo(descriptor);
const pattern = /(?:https:\/\/|git@)?(github|bitbucket|gitlab)?(?::)?(?:\.org|\.com)?(?:\/|:)?([\w-]+\/[\w-]+)(?:\.git)?/;
const match = descriptor.match(pattern);

if (!match) {
throw new InvalidSourceError();
}

const type = (match[1] || 'github') as SCMType;
const repo = match[2];
const url = repoUrlMap[type](repo);

debug(`Valid source: ${url}`);

const { stdout } = await pExec(`git ls-remote ${url}`);
Expand All @@ -57,6 +86,12 @@ export async function getValidConfig(
const archiveUrl = scmArchiveUrls[type](url, sha);
debug(`Archive URL: ${archiveUrl}`);

const ignoreFileUrl = ignoreFileUrls[type](repo, sha);
debug(`Ignore file URL: ${ignoreFileUrl}`);

const ignoreGlobs = await getGlobsToIgnore(ignoreFileUrl);
debug(`Globs to ignore: ${ignoreGlobs}`);

debug('Checking dest is valid');
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const resolvedDest = resolve(dest || url.split('/').pop()!);
Expand All @@ -65,5 +100,5 @@ export async function getValidConfig(
}
debug(`Valid dest: ${resolvedDest}`);

return { resolvedDest, archiveUrl };
return { resolvedDest, archiveUrl, ignoreGlobs };
}
14 changes: 12 additions & 2 deletions src/walk.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import { render } from 'mustache';
import { promises as fsp } from 'fs';
import { join } from 'path';
import { join, resolve } from 'path';
import minimatch from 'minimatch';
import { debug } from './log';

export async function walkAndRender(
dir: string,
templateVariables: { [key: string]: string },
ignoreGlobs: string[],
) {
const files = await fsp.readdir(dir, { withFileTypes: true });

Expand All @@ -14,10 +16,18 @@ export async function walkAndRender(
const filePath = join(dir, file.name);

if (file.isDirectory()) {
return walkAndRender(filePath, templateVariables);
return walkAndRender(filePath, templateVariables, ignoreGlobs);
}

if (file.isFile()) {
if (file.name === '.demsignore') {
return fsp.unlink(filePath);
}

if (ignoreGlobs.some((glob) => minimatch(filePath, resolve(glob)))) {
return Promise.resolve();
}

const fileToRender = await fsp.readFile(filePath, { encoding: 'utf8' });
debug(`Writing file ${filePath}`);
return fsp.writeFile(filePath, render(fileToRender, templateVariables));
Expand Down
32 changes: 22 additions & 10 deletions test/cleanup.ts
Original file line number Diff line number Diff line change
@@ -1,22 +1,34 @@
import { existsSync, rmdirSync, readdirSync, unlinkSync } from 'fs';
import { resolve } from 'path';

function deleteDirectory(path: string) {
if (existsSync(path)) {
const dirents = readdirSync(path, { withFileTypes: true });
dirents.forEach((ent) => {
if (ent.isFile()) {
unlinkSync(resolve(path, ent.name));
}

if (ent.isDirectory()) {
deleteDirectory(resolve(path, ent.name));
}
});

rmdirSync(path);
}
}

/**
* Not an ideal solution, but relatively safe if the tests are not executed in
* parallel, and provides an easier solution than mocking write streams. Note
* that due to performance issues with Jest and parallelism in CircleCI,
* --runInBand is recommended anyway.
*/
export function removeTestDir() {
const dir = resolve('dems-example');
if (existsSync(dir)) {
const dirents = readdirSync(dir, { withFileTypes: true });
dirents.forEach((ent) => {
if (ent.isFile()) {
unlinkSync(resolve(dir, ent.name));
}
});
const fixturePaths = ['dems-example', 'dems-fixture-with-ignores'];

rmdirSync(dir);
}
fixturePaths.forEach((fixturePath) => {
const dir = resolve(fixturePath);
deleteDirectory(dir);
});
}
Loading

0 comments on commit cf7c78b

Please sign in to comment.