Merge pull request #58 from tokyodigitalltd/46-url-crawl

feat: implement url-crawl utility

package.json

@@ -26,23 +26,26 @@
     "@11ty/eleventy-cache-assets": "^2.3.0",
     "@imgix/js-core": "^3.1.3",
     "@justinribeiro/lite-youtube": "^0.9.1",
+    "@mdn/browser-compat-data": "5.2.x",
     "@swc/html": "0.0.18",
     "async-transforms": "^1.0.9",
     "browser-media-mime-type": "^1.0.0",
+    "chalk": "4.1.2",
     "common-tags": "^1.8.0",
     "csso": "5.0.5",
+    "express": "4.18.2",
     "fast-glob": "3.2.12",
     "js-yaml": "4.1.0",
     "lit-element": "^2.5.1",
     "lodash.get": "4.4.2",
     "lodash.set": "4.3.2",
     "markdown-it": "^12.3.2",
+    "node-mocks-http": "1.12.1",
     "nunjucks": "3.2.3",
     "piscina": "^3.2.0",
     "purgecss": "5.0.0",
     "querystring": "^0.2.1",
-    "striptags": "^3.1.1",
-    "@mdn/browser-compat-data": "5.2.x"
+    "striptags": "^3.1.1"
   },
   "peerDependencies": {
     "@mdn/browser-compat-data": "5.2.x"

types/utils/url-crawl/ScanError.d.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type ScanError = {
+  path: string;
+  tag: string;
+  statusCode: number;
+  summary: string;
+  parent: string | null;
+};

types/utils/url-crawl/UrlCrawlOptions.d.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type UrlCrawlOptions = {
+  shouldNormalizeTrailingSlash?: boolean;
+  shouldDisableDuplicateUrls?: boolean;
+  onErrorOutput?: (error: ScanError) => string;
+  handlers?: import('express').RequestHandler[];
+  shouldDetectSuperstatic?: boolean;
+};

types/utils/url-crawl/index.d.ts

@@ -0,0 +1,18 @@
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export * from './ScanError';
+export * from './UrlCrawlOptions';