Merge branch 'main' into backstage-1.18.4

packages/app/package.json

@@ -19,6 +19,7 @@
     "@backstage/plugin-catalog-graph": "^0.2.36",
     "@backstage/plugin-catalog-import": "^0.10.0",
     "@backstage/plugin-catalog-react": "^1.8.4",
+    "@backstage/plugin-catalog-unprocessed-entities": "^0.1.3",
     "@backstage/plugin-github-actions": "^0.6.5",
     "@backstage/plugin-graphiql": "^0.2.54",
     "@backstage/plugin-org": "^0.6.14",

packages/app/src/App.tsx

@@ -43,6 +43,7 @@ import { auth0AuthApiRef } from './internal';
 import Star from '@material-ui/icons/Star';
 import { githubAuthApiRef } from '@backstage/core-plugin-api';
 import { SecretsContextProvider } from '@backstage/plugin-scaffolder-react';
+import { CatalogUnprocessedEntitiesPage } from '@backstage/plugin-catalog-unprocessed-entities';
 
 const app = createApp({
   apis,
@@ -87,6 +88,10 @@ const routes = (
     >
       {entityPage}
     </Route>
+    <Route
+      path="/catalog-unprocessed-entities"
+      element={<CatalogUnprocessedEntitiesPage />}
+    />
     <Route path="/docs" element={<TechDocsIndexPage />}>
       <DefaultTechDocsHome />
     </Route>

packages/backend/package.json

@@ -29,6 +29,7 @@
     "@backstage/plugin-auth-node": "^0.3.2",
     "@backstage/plugin-catalog-backend": "^1.13.3",
     "@backstage/plugin-catalog-backend-module-github": "^0.4.3",
+    "@backstage/plugin-catalog-backend-module-unprocessed": "^0.3.2",
     "@backstage/plugin-catalog-graph": "^0.2.36",
     "@backstage/plugin-permission-common": "^0.7.8",
     "@backstage/plugin-permission-node": "^0.7.16",

packages/backend/src/plugins/catalog.ts

@@ -1,28 +1,37 @@
-import {
-  CatalogBuilder
-} from '@backstage/plugin-catalog-backend';
+import { CatalogBuilder } from '@backstage/plugin-catalog-backend';
 import { ScaffolderEntitiesProcessor } from '@backstage/plugin-scaffolder-backend';
+import { UnprocessedEntitiesModule } from '@backstage/plugin-catalog-backend-module-unprocessed';
 import { IncrementalCatalogBuilder } from '@frontside/backstage-plugin-incremental-ingestion-backend';
 import { Router } from 'express';
 import { PluginEnvironment } from '../types';
 
 export default async function createPlugin(
   env: PluginEnvironment,
 ): Promise<Router> {
-
   const builder = CatalogBuilder.create(env);
+
   // incremental builder receives builder because it'll register
-  // incremental entity providers with the builder 
-  const incrementalBuilder = await IncrementalCatalogBuilder.create(env, builder);
-
+  // incremental entity providers with the builder
+  const incrementalBuilder = await IncrementalCatalogBuilder.create(
+    env,
+    builder,
+  );
+
   builder.addProcessor(new ScaffolderEntitiesProcessor());
 
   const { processingEngine, router } = await builder.build();
 
-  // this has to run after `await builder.build()` so ensure that catalog migrations are completed 
-  // before incremental builder migrations are executed 
+  // this has to run after `await builder.build()` so ensure that catalog migrations are completed
+  // before incremental builder migrations are executed
   await incrementalBuilder.build();
 
+  const unprocessed = new UnprocessedEntitiesModule(
+    await env.database.getClient(),
+    router,
+  );
+
+  unprocessed.registerRoutes();
+
   await processingEngine.start();
 
   return router;

packages/backend/src/plugins/graphql.ts

@@ -1,4 +1,4 @@
-import { Catalog } from '@frontside/backstage-plugin-graphql-backend-module-catalog';
+import { Catalog, createCatalogLoader } from '@frontside/backstage-plugin-graphql-backend-module-catalog';
 import { createRouter } from '@frontside/backstage-plugin-graphql-backend';
 import { Router } from 'express';
 import { PluginEnvironment } from '../types';
@@ -9,6 +9,7 @@ export default async function createPlugin(
 ): Promise<Router> {
   return await createRouter({
     logger: env.logger,
-    modules: [myModule, await Catalog()],
+    modules: [myModule, Catalog()],
+    loaders: { ...createCatalogLoader(env.catalog) },
   });
 }

packages/backend/src/types.ts

@@ -1,17 +1,17 @@
-import { Logger } from 'winston';
-import { Config } from '@backstage/config';
-import {
+import type { Logger } from 'winston';
+import type { Config } from '@backstage/config';
+import type {
   DatabaseManager,
   PluginCacheManager,
   PluginDatabaseManager,
   PluginEndpointDiscovery,
   TokenManager,
   UrlReader,
 } from '@backstage/backend-common';
-import { PluginTaskScheduler } from '@backstage/backend-tasks';
-import { IdentityApi } from '@backstage/plugin-auth-node';
-import { PermissionEvaluator } from '@backstage/plugin-permission-common';
-import { CatalogClient } from '@backstage/catalog-client';
+import type { PluginTaskScheduler } from '@backstage/backend-tasks';
+import type { IdentityApi } from '@backstage/plugin-auth-node';
+import type { PermissionEvaluator } from '@backstage/plugin-permission-common';
+import type { CatalogClient } from '@backstage/catalog-client';
 
 export type PluginEnvironment = {
   logger: Logger;
@@ -25,5 +25,5 @@ export type PluginEnvironment = {
   scheduler: PluginTaskScheduler;
   permissions: PermissionEvaluator;
   identity: IdentityApi;
-  catalog: CatalogClient
+  catalog: CatalogClient;
 };

yarn.lock

@@ -1966,9 +1966,9 @@
     regenerator-runtime "^0.14.0"
 
 "@babel/runtime@^7.20.6":
-  version "7.23.1"
-  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.23.1.tgz#72741dc4d413338a91dcb044a86f3c0bc402646d"
-  integrity sha512-hC2v6p8ZSI/W0HUzh3V8C5g+NwSKzKPtJwSpTjwl0o297GP9+ZLQSkdvHz46CM3LqyoXxq+5G9komY+eSqSO0g==
+  version "7.23.2"
+  resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.23.2.tgz#062b0ac103261d68a966c4c7baf2ae3e62ec3885"
+  integrity sha512-mM8eg4yl5D6i3lu2QKPuPH4FArvJ8KhTofbE7jwMUv9KX5mBvwPAqnV3MlyBNqdp9RyRKP6Yck8TrfYrPvX3bg==
   dependencies:
     regenerator-runtime "^0.14.0"
 
@@ -2764,6 +2764,17 @@
     "@backstage/plugin-catalog-node" "^1.4.6"
     "@backstage/plugin-scaffolder-common" "^1.4.1"
 
+"@backstage/plugin-catalog-backend-module-unprocessed@^0.3.2":
+  version "0.3.2"
+  resolved "https://registry.yarnpkg.com/@backstage/plugin-catalog-backend-module-unprocessed/-/plugin-catalog-backend-module-unprocessed-0.3.2.tgz#67bc2b7fc21d0864c1cf9b5c07189e61a9544254"
+  integrity sha512-C5GUxA4jHalDKK8htgxpqhsa+qARYMLWni0y6tJ0Zz5H7nxbpP04hjkHaqjlFN4SoqucP1s3+nIVWaVThrSQvg==
+  dependencies:
+    "@backstage/backend-plugin-api" "^0.6.5"
+    "@backstage/catalog-model" "^1.4.2"
+    "@backstage/plugin-auth-node" "^0.3.2"
+    express-promise-router "^4.1.1"
+    knex "^2.4.2"
+
 "@backstage/plugin-catalog-backend@^1.13.3":
   version "1.13.3"
   resolved "https://registry.yarnpkg.com/@backstage/plugin-catalog-backend/-/plugin-catalog-backend-1.13.3.tgz#80ad09dd6d5a0feb4bcfd4e61edc3b0f6b83aa5d"
@@ -2912,6 +2923,22 @@
     yaml "^2.0.0"
     zen-observable "^0.10.0"
 
+"@backstage/plugin-catalog-unprocessed-entities@^0.1.3":
+  version "0.1.3"
+  resolved "https://registry.yarnpkg.com/@backstage/plugin-catalog-unprocessed-entities/-/plugin-catalog-unprocessed-entities-0.1.3.tgz#a84e4f257c2bbabe4e7a3f0d6610eb6298e225ee"
+  integrity sha512-a612yty5DCJcjOj5mXl5r0zEDMkJT3v2YLsDLdIga9LEqSJ/VU7B3UnZjUnO7dm3ZJpQerUVW0vvKv3jFbKujw==
+  dependencies:
+    "@backstage/catalog-model" "^1.4.2"
+    "@backstage/core-components" "^0.13.5"
+    "@backstage/core-plugin-api" "^1.6.0"
+    "@backstage/errors" "^1.2.2"
+    "@backstage/theme" "^0.4.2"
+    "@material-ui/core" "^4.9.13"
+    "@material-ui/icons" "^4.9.1"
+    "@material-ui/lab" "^4.0.0-alpha.60"
+    "@types/react" "^16.13.1 || ^17.0.0"
+    react-use "^17.2.4"
+
 "@backstage/plugin-catalog@^1.13.0":
   version "1.13.0"
   resolved "https://registry.yarnpkg.com/@backstage/plugin-catalog/-/plugin-catalog-1.13.0.tgz#fd2a0288eacad8209b80792e29d62571da51b799"
@@ -6638,7 +6665,7 @@
     prop-types "^15.7.2"
     react-is "^16.8.0 || ^17.0.0"
 
-"@material-ui/[email protected]", "@material-ui/lab@^4.0.0-alpha.57":
+"@material-ui/[email protected]", "@material-ui/lab@^4.0.0-alpha.57", "@material-ui/lab@^4.0.0-alpha.60":
   version "4.0.0-alpha.61"
   resolved "https://registry.yarnpkg.com/@material-ui/lab/-/lab-4.0.0-alpha.61.tgz#9bf8eb389c0c26c15e40933cc114d4ad85e3d978"
   integrity sha512-rSzm+XKiNUjKegj8bzt5+pygZeckNLOr+IjykH8sYdVk7dE9y2ZuUSofiMV2bJk3qU+JHwexmw+q0RyNZB9ugg==
@@ -14556,7 +14583,7 @@ express-openapi-validator@^5.0.4:
     ono "^7.1.3"
     path-to-regexp "^6.2.0"
 
-express-promise-router@^4.1.0:
+express-promise-router@^4.1.0, express-promise-router@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/express-promise-router/-/express-promise-router-4.1.1.tgz#8fac102060b9bcc868f84d34fbb12fd8fa494291"
   integrity sha512-Lkvcy/ZGrBhzkl3y7uYBHLMtLI4D6XQ2kiFg9dq7fbktBch5gjqJ0+KovX0cvCAvTJw92raWunRLM/OM+5l4fA==
@@ -18173,7 +18200,7 @@ knex@^1.0.3:
     tarn "^3.0.2"
     tildify "2.0.0"
 
-knex@^2.0.0, knex@^2.3.0:
+knex@^2.0.0, knex@^2.3.0, knex@^2.4.2:
   version "2.5.1"
   resolved "https://registry.yarnpkg.com/knex/-/knex-2.5.1.tgz#a6c6b449866cf4229f070c17411f23871ba52ef9"
   integrity sha512-z78DgGKUr4SE/6cm7ku+jHvFT0X97aERh/f0MUKAKgFnwCYBEW4TFBqtHWFYiJFid7fMrtpZ/gxJthvz5mEByA==