Introduce Twig scanner into Analytics

.github/workflows/test.yaml

@@ -64,3 +64,12 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           file: ./packages/web-react/.coverage/lcov.info
           flag-name: web-react
+
+      - name: Publish Analytics Package Code Coverage
+        # When Nx hits its cloud cache, there is no generated coverage to sent, e.g. do not let this crash
+        if: ${{ hashFiles('./packages/analytics/.coverage/lcov.info') != '' }}
+        uses: coverallsapp/[email protected]
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          file: ./packages/analytics/.coverage/lcov.info
+          flag-name: analytics

packages/analytics/README.md

@@ -17,7 +17,7 @@ yarn add -D @lmc-eu/spirit-analytics
 
 ## Usage
 
-Running the command without arguments will use working directory as a root and parse entire directory structure from this point
+Running the command without arguments will use the working directory as a root and parse the entire directory structure from this point.
 
 ```shell
 spirit-analytics
@@ -29,19 +29,61 @@ You can parse only specific directories from the project with `--source` argumen
 spirit-analytics --source ./frontend
 ```
 
-By default the output will be saved into `.scanner` directory, but you can specify the path with `--output` argument:
+By default, the output will be saved into the `.scanner` directory, but you can specify the path with the `--output` argument:
 
 ```shell
 spirit-analytics --output path/to/folder
 ```
 
-The [react-scanner] requires a [config file][react-scanner-config] to make it work, spirit-analytics has a default config inside, but if you will need, you can use your own config:
+The [react-scanner] requires a [config file][react-scanner-config] to make it work, `spirit-analytics` has a default config inside, but if you need to, you can use your own config:
 
 ```shell
 spirit-analytics --config path/to/config
 ```
 
+You can easily switch from React scanner to Twig scanner using `type` argument. By default, both scanners will be used.
+
+```shell
+spirit-analytics --type react
+```
+
 You can run `spirit-analytics --help` to get a list of available options and examples.
 
+## Configuration
+
+You can provide your own configuration file in the following format:
+
+```js
+export default {
+  react: {
+    // react-scanner config; @see https://www.npmjs.com/package/react-scanner#config-file
+  },
+  twig: {
+    crawlFrom: './',
+    exclude: ['node_modules', 'dist', 'build', 'coverage', 'public', 'vendor', 'storybook-static'],
+    configFile: './config/spirit-web-twig.yml',
+    outputFile: './.scanner/adoption-data-twig.json',
+    coreComponentsPath: './vendor/lmc/spirit-web-twig-bundle/src/Resources/twig-components',
+  },
+};
+```
+
+### Configuration Options
+
+#### React
+
+👉 [`react-scanner` Configuration Options][react-scanner-config-options]
+
+#### Twig
+
+| Option             | Type                  | Description                                                                                                                                                                                                                                                                                                                                   |
+| ------------------ | --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| crawlFrom          | `string`              | The path of the directory to start crawling from.                                                                                                                                                                                                                                                                                             |
+| exclude            | `array` or `function` | Each array item should be a string or a regex. When crawling, if directory name matches exactly the string item or matches the regex item, it will be excluded from crawling. For more complex scenarios, exclude can be a a function that accepts a directory name and should return true if the directory should be excluded from crawling. |
+| configFile         | `string`              | Path to the local `spirit-web-twig.yml` configuration file.                                                                                                                                                                                                                                                                                   |
+| outputFile         | `string`              | Path to the file where the result of the analysis will be stored.                                                                                                                                                                                                                                                                             |
+| coreComponentsPath | `string`              | Path to the directory where are core Spirit components installed.                                                                                                                                                                                                                                                                             |
+
 [react-scanner]: https://github.com/moroshko/react-scanner
 [react-scanner-config]: https://github.com/moroshko/react-scanner#config-file
+[react-scanner-config-options]: https://www.npmjs.com/package/react-scanner#config-options

packages/analytics/config/jest.config.js

@@ -31,11 +31,15 @@ const config = {
 
   // An array of regexp pattern strings that are matched against all file paths before executing the test.
   // https://jestjs.io/docs/configuration#coveragepathignorepatterns-arraystring
-  coveragePathIgnorePatterns: ['__fixtures__'],
+  coveragePathIgnorePatterns: ['__fixtures__', 'bin'],
 
   // A list of reporter names that Jest uses when writing coverage reports. Any istanbul reporter can be used.
   // https://jestjs.io/docs/configuration#coveragereporters-arraystring--string-options
   coverageReporters: ['text', 'text-summary', ['lcov', { projectRoot: '../../' }]],
+
+  // An array of regexp pattern strings that are matched against all source file paths before transformation.
+  // https://jestjs.io/docs/configuration#transformignorepatterns-arraystring
+  transformIgnorePatterns: ['<rootDir>/../../node_modules/zx'],
 };
 
 export default config;

packages/analytics/config/tsup.config.ts

@@ -1,9 +1,9 @@
 import { defineConfig } from 'tsup';
 
 export default defineConfig({
-  entry: ['src/**/*.ts', '!src/**/__tests__/*.ts', '!src/**/*.test.ts'],
+  entry: ['src/index.ts', 'src/spirit-analytics.config.ts'],
   format: ['esm', 'cjs'],
   dts: true,
-  splitting: false,
+  splitting: true,
   clean: true,
 });

packages/analytics/package.json

@@ -19,7 +19,7 @@
     "access": "public",
     "directory": "dist"
   },
-  "main": "./index.js",
+  "main": "./index.cjs",
   "module": "./index.js",
   "types": "./index.d.ts",
   "bin": "./bin/spirit-analytics.js",
@@ -34,7 +34,7 @@
   "scripts": {
     "prebuild": "shx rm -rf dist && shx mkdir -p dist",
     "build": "tsup --config ./config/tsup.config.ts",
-    "postbuild": "shx cp -r package.json README.md dist/",
+    "postbuild": "shx cp -r package.json README.md src/bin dist/",
     "start": "tsup src/index.ts --watch",
     "types": "tsc",
     "lint": "eslint ./",

packages/analytics/src/__tests__/runner.test.ts

@@ -0,0 +1,59 @@
+import scanner from 'react-scanner';
+import { path } from 'zx';
+import { runner } from '../runner';
+import { RunnerConfig } from '../types';
+
+jest.mock('../scanners/twigScanner', () => {
+  return jest.fn(() => Promise.resolve({ test: 'test' }));
+});
+
+describe('runner', () => {
+  jest.spyOn(path, 'resolve').mockImplementation((...args: string[]) => args.join('/'));
+
+  it('should return tracked data for react type', async () => {
+    const config = {
+      react: {},
+      twig: {},
+    } as RunnerConfig;
+    const source = '/path/to/source';
+    const type = 'react';
+    jest.spyOn(scanner, 'run').mockResolvedValue({ test: 'test' });
+
+    const trackedData = await runner(config, source, type);
+
+    expect(trackedData.spiritVersion).toBeDefined();
+    expect(trackedData.trackedData.react).toEqual({ test: 'test' });
+    expect(trackedData.trackedData.twig).toEqual({});
+  });
+
+  it('should return tracked data for twig type', async () => {
+    const config = {
+      react: {},
+      twig: {},
+    } as RunnerConfig;
+    const source = '/path/to/source';
+    const type = 'twig';
+
+    const trackedData = await runner(config, source, type);
+
+    expect(trackedData.spiritVersion).toBeDefined();
+    expect(trackedData.trackedData.react).toEqual({});
+    expect(trackedData.trackedData.twig).toEqual({ test: 'test' });
+  });
+
+  it('should return tracked data for both react and twig types', async () => {
+    const config = {
+      react: {},
+      twig: {},
+    } as RunnerConfig;
+    const source = '/path/to/source';
+    const type = null;
+    jest.spyOn(scanner, 'run').mockResolvedValue({ test: 'test' });
+
+    const trackedData = await runner(config, source, type);
+
+    expect(trackedData.spiritVersion).toBeDefined();
+    expect(trackedData.trackedData.react).toEqual({ test: 'test' });
+    expect(trackedData.trackedData.twig).toEqual({ test: 'test' });
+  });
+});

packages/analytics/src/bin/spirit-analytics.js

@@ -0,0 +1,5 @@
+#!/usr/bin/env node
+// eslint-disable-next-line import/no-unresolved
+import { cli } from '../index.js';
+
+cli(process.argv);

packages/analytics/src/cli.ts

@@ -1,10 +1,11 @@
 import sade from 'sade';
 import { fs, path } from 'zx';
-import { errorMessage, infoMessage } from './helpers';
-import scanner from './scanner';
 import { ROOT_PATH } from './constants';
+import { _dirname, errorMessage, infoMessage } from './helpers';
+import scanner from './scanner';
+import { ScannerType } from './types';
 
-const packageJson = fs.readJsonSync(path.resolve(__dirname, '../package.json'));
+const packageJson = fs.readJsonSync(path.resolve(_dirname, './package.json'));
 
 export default async function cli(args: string[]) {
   sade('analytics', true)
@@ -16,26 +17,45 @@ export default async function cli(args: string[]) {
     .example('-o path/to/folder')
     .option('-c --config', 'Path to scanner config')
     .example('-c path/to/scanner.config.js')
-    .action(({ output, config, source }) => {
+    .option('-t --type', 'Type of scanner')
+    .example('-t react')
+    .action(async ({ output, config, source, type }) => {
+      let selectedConfig;
+
       if (config && !fs.existsSync(config)) {
         errorMessage('Could not find config file');
         process.exit(1);
+      } else if (config && fs.existsSync(config)) {
+        infoMessage(`Using provided config file: ${config}`);
+        selectedConfig = path.resolve(process.cwd(), config);
       } else {
         infoMessage('Using default config file');
-        // eslint-disable-next-line no-param-reassign
-        config = path.resolve(__dirname, '../react-scanner.config.js');
+        selectedConfig = path.resolve(_dirname, './spirit-analytics.config.js');
       }
 
       if (output && !fs.existsSync(output)) {
         errorMessage('Path does not exists');
         process.exit(1);
       }
 
+      const { default: loadedConfig } = await import(selectedConfig);
+
+      let selectedType: ScannerType = null;
+
+      if (type) {
+        selectedType = type;
+      }
+
       if (source) {
-        scanner({ source, outputPath: output, config });
+        scanner({ source, outputPath: output, config: loadedConfig, type: selectedType });
         infoMessage(`Start scanning: ${source}`);
       } else {
-        scanner({ source: ROOT_PATH, outputPath: output, config });
+        scanner({
+          source: path.resolve(process.cwd(), ROOT_PATH),
+          outputPath: output,
+          config: loadedConfig,
+          type: selectedType,
+        });
         infoMessage('Start scanning from default scope');
       }
     })

packages/analytics/src/constants.ts

@@ -1,5 +1,7 @@
-import { path } from 'zx';
-
-export const ROOT_PATH = path.resolve(__dirname, '..');
-export const OUTPUT_DIR = path.resolve(__dirname, '../.scanner');
 export const OUTPUT_FILENAME_PREFIX = 'spirit-analytics';
+export const OUTPUT_DIR = '.scanner';
+export const ROOT_PATH = '.';
+export const REACT_OUTPUT_FILE = 'adoption-data-react.json';
+export const TWIG_OUTPUT_FILE = './.scanner/adoption-data-twig.json';
+export const TWIG_CORE_COMPONENTS_PATH = './vendor/lmc/spirit-web-twig-bundle/src/Resources/twig-components';
+export const TWIG_CONFIG_FILE = './config/spirit-web-twig.yml';

packages/analytics/src/helpers/__tests__/path.test.ts

@@ -15,29 +15,6 @@ jest.mock('../../constants', () => ({
 
 describe('path', () => {
   describe('getOutputPath', () => {
-    let originalDirname: string;
-    let originalFilename: string;
-
-    beforeAll(() => {
-      // Store the original values of __dirname and __filename
-      originalDirname = global.__dirname;
-      originalFilename = global.__filename;
-
-      // Mock the global __dirname and __filename
-      global.__dirname = 'mocked-global-dirname';
-      global.__filename = 'mocked-global-filename';
-    });
-
-    afterAll(() => {
-      // Restore the original values of __dirname and __filename
-      global.__dirname = originalDirname;
-      global.__filename = originalFilename;
-    });
-
-    afterEach(() => {
-      jest.clearAllMocks();
-    });
-
     it('should use the provided outputPath if available', () => {
       const outputPath = '/path/to/output';
       const name = 'example';

packages/analytics/src/helpers/index.ts

@@ -1,7 +1,7 @@
+import { timestamp } from './date';
 import { errorMessage, infoMessage } from './message';
 import { getModuleName } from './moduleName';
-import { getOutputPath } from './path';
+import { _dirname, getOutputPath } from './path';
 import { getVersions } from './versions';
-import { timestamp } from './date';
 
-export { getModuleName, infoMessage, errorMessage, getOutputPath, getVersions, timestamp };
+export { _dirname, errorMessage, getModuleName, getOutputPath, getVersions, infoMessage, timestamp };

packages/analytics/src/helpers/path.ts

@@ -1,6 +1,9 @@
+import filedirname from 'filedirname';
 import { path } from 'zx';
 import { OUTPUT_DIR, OUTPUT_FILENAME_PREFIX } from '../constants';
 
+export const [_filename, _dirname] = filedirname();
+
 export const getOutputPath = (outputPath: string, name: string) => {
   const outputFilename = `${OUTPUT_FILENAME_PREFIX}-${name}.json`;
 

packages/analytics/src/index.ts

@@ -1,9 +1,9 @@
 import cli from './cli';
-import scanner from './scanner';
-import spiritAdoptionProcessor from './processors/spiritAdoptionProcessor';
-import reactScannerConfig from './react-scanner.config';
-import * as helpers from './helpers';
 import * as constants from './constants';
+import * as helpers from './helpers';
+import spiritAdoptionProcessor from './processors/spiritAdoptionProcessor';
+import scanner from './scanner';
+import reactScannerConfig from './scanners/react-scanner.config';
 import * as types from './types';
 
-export { cli, scanner, spiritAdoptionProcessor, helpers, constants, reactScannerConfig, types };
+export { cli, constants, helpers, reactScannerConfig, scanner, spiritAdoptionProcessor, types };

packages/analytics/src/processors/spiritAdoptionProcessor.ts

@@ -1,11 +1,11 @@
 import { path } from 'zx';
-import { getModuleName } from '../helpers';
-import { ROOT_PATH } from '../constants';
+import { OUTPUT_DIR, REACT_OUTPUT_FILE, ROOT_PATH } from '../constants';
+import { _dirname, getModuleName } from '../helpers';
 import { Component } from '../types';
 
-const OUTPUT_FILE = path.resolve(__dirname, '../.scanner/adoption-data-react.json');
+const OUTPUT_FILE = path.resolve(process.cwd(), `${OUTPUT_DIR}/${REACT_OUTPUT_FILE}`);
 
-const getRelativePath = (absolutePath: string) => path.relative(ROOT_PATH, absolutePath);
+const getRelativePath = (absolutePath: string) => path.relative(path.resolve(_dirname, ROOT_PATH), absolutePath);
 
 interface ForEachComponentOptions {
   componentName: string;