ci: typoチェック

yamadashy · Oct 11, 2024 · 2a3e8ca · 2a3e8ca
1 parent 297cca1
commit 2a3e8ca
Show file tree

Hide file tree

Showing 11 changed files with 102 additions and 437 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -13,6 +13,7 @@ on:
 
 jobs:
   lint-biome:
+    name: Lint Biome
     runs-on: ubuntu-24.04
     timeout-minutes: 10
     steps:
@@ -25,6 +26,7 @@ jobs:
     - run: yarn lint-biome && git diff --exit-code
 
   lint-ts:
+    name: Lint TypeScript
     runs-on: ubuntu-24.04
     timeout-minutes: 10
     steps:
@@ -37,6 +39,7 @@ jobs:
     - run: yarn lint-ts
 
   lint-secretlint:
+    name: Lint Secretlint
     runs-on: ubuntu-24.04
     timeout-minutes: 10
     steps:
@@ -48,7 +51,8 @@ jobs:
     - run: yarn install
     - run: yarn lint-secretlint
 
-  validate-renovate-config:
+  lint-renovate-config:
+    name: Lint Renovate config
     runs-on: ubuntu-24.04
     timeout-minutes: 10
     steps:
@@ -59,7 +63,15 @@ jobs:
     - name: Validate Renovate config
       run: npx --yes --package renovate -- renovate-config-validator --strict
 
+  check-typo:
+    name: Check typos
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: crate-ci/[email protected]
+
   test:
+    name: Test
     runs-on: ubuntu-24.04
     timeout-minutes: 10
     steps:
@@ -72,6 +84,7 @@ jobs:
     - run: yarn test-coverage
 
   generate:
+    name: Generate feed and site
     runs-on: ubuntu-24.04
     timeout-minutes: 20
     steps:

diff --git a/.typos.toml b/.typos.toml
@@ -0,0 +1,4 @@
+[files]
+extend-exclude = [
+  "src/resources/feed-info-list.ts"
+]
diff --git a/package.json b/package.json
@@ -43,7 +43,6 @@
     "@types/http-server": "^0.12.1",
     "@types/node": "^20.10.4",
     "@types/open-graph-scraper": "^4.8.1",
-    "@types/request": "^2.48.8",
     "@vitest/coverage-v8": "^2.0.3",
     "async-retry": "^1.3.3",
     "await-to-js": "^3.0.0",
@@ -59,7 +58,6 @@
     "log4js": "^6.9.1",
     "open-graph-scraper": "^6.5.1",
     "renovate": "^37.253.1",
-    "request": "^2.88.2",
     "rss-parser": "^3.12.0",
     "secretlint": "^8.3.0",
     "sharp-ico": "^0.1.5",

diff --git a/src/feed/generate-feed-command.ts b/src/feed/generate-feed-command.ts
@@ -6,6 +6,7 @@ import { FeedGenerator } from './utils/feed-generator';
 import { ImagePrecacher as FeedImagePrecacher } from './utils/feed-image-precacher';
 import { FeedStorer } from './utils/feed-storer';
 import { FeedValidator } from './utils/feed-validator';
+import { logger } from './utils/logger';
 
 const dirName = url.fileURLToPath(new URL('.', import.meta.url));
 
@@ -43,19 +44,6 @@ const feedImagePrecacher = new FeedImagePrecacher();
     MAX_FEED_CONTENT_LENGTH,
   );
 
-  // まとめフィードのバリデーション。エラーならすぐに終了する
-  try {
-    await feedValidator.assertFeed(generateFeedsResult.aggregatedFeed);
-    await feedValidator.assertXmlFeed('atom', generateFeedsResult.feedDistributionSet.atom);
-    await feedValidator.assertXmlFeed('rss', generateFeedsResult.feedDistributionSet.rss);
-  } catch (e) {
-    const error = new Error('Failed to validate feed', {
-      cause: e,
-    });
-    console.error(error);
-    throw error;
-  }
-
   // ファイル出力
   try {
     await feedStorer.storeFeeds(
@@ -89,4 +77,21 @@ const feedImagePrecacher = new FeedImagePrecacher();
     console.error(error);
     throw error;
   }
+
+  // 最後にまとめフィードのバリデーション
+  try {
+    logger.info('フィードのバリデーション開始');
+
+    await feedValidator.assertFeed(generateFeedsResult.aggregatedFeed);
+    await feedValidator.assertXmlFeed('atom', generateFeedsResult.feedDistributionSet.atom);
+    await feedValidator.assertXmlFeed('rss', generateFeedsResult.feedDistributionSet.rss);
+
+    logger.info('フィードのバリデーション完了');
+  } catch (e) {
+    const error = new Error('Failed to validate feed', {
+      cause: e,
+    });
+    console.error(error);
+    throw error;
+  }
 })();
diff --git a/src/feed/register-index-command.ts b/src/feed/register-index-command.ts
@@ -1,5 +1,4 @@
 import { google } from 'googleapis';
-import * as request from 'request';
 import constants from '../common/constants';
 import { sleep } from './utils/common-util';
 import type { BlogFeed } from './utils/feed-storer';
@@ -45,30 +44,29 @@ jwtClient.authorize(async (err, tokens) => {
   }
 
   for (const indexTargetUrl of indexTargetUrls) {
-    const options: request.CoreOptions = {
+    const options = {
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',
+        Authorization: `Bearer ${tokens.access_token || ''}`,
       },
-      auth: { bearer: tokens.access_token || '' },
-      json: {
+      body: JSON.stringify({
         url: indexTargetUrl,
         type: 'URL_UPDATED',
-      },
+      }),
     };
 
-    await request.post(GOOGLE_INDEXING_API_END_POINT, options, async (error, response, body) => {
-      if (error) {
-        console.error(error);
-        return;
-      }
-      if (response.statusCode !== 200) {
-        console.error(response.statusCode, body);
-        return;
+    try {
+      const response = await fetch(GOOGLE_INDEXING_API_END_POINT, options);
+      if (!response.ok) {
+        console.error(response.status, await response.text());
+        continue;
       }
-
+      const body = await response.json();
       console.log(`[index api] success! url: ${body.urlNotificationMetadata.url}`);
-    });
+    } catch (error) {
+      console.error(error);
+    }
 
     await sleep(1000);
   }

diff --git a/src/feed/utils/common-util.ts b/src/feed/utils/common-util.ts
@@ -32,16 +32,6 @@ export const removeInvalidUnicode = (text: string) => {
   return text.replace(/[\x00-\x1F\x7F-\x9F]/g, '');
 };
 
-export const escapeTextForXml = (text: string) => {
-  let escapedText = text;
-
-  if (text.includes('&')) {
-    escapedText = text.replace(/&/g, '&amp;');
-  }
-
-  return escapedText;
-};
-
 export const isValidHttpUrl = (url: string) => {
   let urlObject: URL;
 

diff --git a/src/feed/utils/feed-crawler.ts b/src/feed/utils/feed-crawler.ts
@@ -15,6 +15,7 @@ import {
   removeInvalidUnicode,
   urlRemoveQueryParams,
 } from './common-util';
+import { FeedValidator } from './feed-validator';
 import { logger } from './logger';
 
 export type CustomOgObject = OgObject & {
@@ -44,6 +45,7 @@ export interface ClawlFeedsResult {
 
 export class FeedCrawler {
   private rssParser;
+  private feedValidator;
 
   constructor() {
     this.rssParser = new RssParser({
@@ -53,6 +55,7 @@ export class FeedCrawler {
         'user-agent': constants.requestUserAgent,
       },
     });
+    this.feedValidator = new FeedValidator();
   }
 
   public async crawlFeeds(
@@ -113,6 +116,10 @@ export class FeedCrawler {
             }
 
             const feedData = await response.text();
+
+            // バリデーション
+            await this.feedValidator.assertXmlFeed('fetched-feed', feedData);
+
             return this.rssParser.parseString(feedData) as Promise<CustomRssParserFeed>;
           }),
         );

diff --git a/src/feed/utils/feed-generator.ts b/src/feed/utils/feed-generator.ts
@@ -1,6 +1,6 @@
 import { Feed, type FeedOptions } from 'feed';
 import constants from '../../common/constants.js';
-import { escapeTextForXml, textToMd5Hash, textTruncate } from './common-util';
+import { textToMd5Hash, textTruncate } from './common-util';
 import type { CustomRssParserItem, FeedItemHatenaCountMap, OgObjectMap } from './feed-crawler';
 import { logger } from './logger';
 
@@ -14,6 +14,13 @@ export interface GenerateFeedResult {
   aggregatedFeed: Feed;
   feedDistributionSet: FeedDistributionSet;
 }
+const escapeTextForXml = (text: string) => {
+  return text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
+};
+
+const escapeAmpersand = (text: string) => {
+  return text.replace(/&/g, '&amp;');
+};
 
 export class FeedGenerator {
   public generateFeeds(
@@ -35,8 +42,8 @@ export class FeedGenerator {
       aggregatedFeed,
       feedDistributionSet: {
         // 出力されているXMLで & がエスケープされていないのでパッチ対応
-        atom: escapeTextForXml(aggregatedFeed.atom1()),
-        rss: escapeTextForXml(aggregatedFeed.rss2()),
+        atom: escapeAmpersand(aggregatedFeed.atom1()),
+        rss: escapeAmpersand(aggregatedFeed.rss2()),
         json: aggregatedFeed.json1(),
       },
     };
@@ -70,9 +77,12 @@ export class FeedGenerator {
       const feedItemContent = (feedItem.summary || feedItem.contentSnippet || '').replace(/(\n|\t+|\s+)/g, ' ');
 
       const ogObject = feedItemOgObjectMap.get(feedItem.link);
-      // 配列になっているが2つ目以降を使う理由もないので0を使う
       const ogImage = ogObject?.customOgImage;
 
+      if (ogImage && ogImage.alt) {
+        ogImage.alt = escapeTextForXml(ogImage.alt);
+      }
+
       // 日付がないものは入れない
       if (!feedItem.isoDate) {
         logger.warn('[feed-item] フィードの日付がありません。', feedItem.isoDate, feedItem.title);
@@ -83,20 +93,20 @@ export class FeedGenerator {
         id: feedItemId,
         guid: feedItemId,
         // 「記事タイトル | ブログ名」の形にする。タイトルだけでどの企業かわかるように
-        title: `${feedItem.title} | ${feedItem.blogTitle}`,
-        description: textTruncate(feedItemContent, maxFeedDescriptionLength),
-        content: textTruncate(feedItemContent, maxFeedContentLength),
+        title: escapeTextForXml(`${feedItem.title} | ${feedItem.blogTitle}`),
+        description: escapeTextForXml(textTruncate(feedItemContent, maxFeedDescriptionLength)),
+        content: escapeTextForXml(textTruncate(feedItemContent, maxFeedContentLength)),
         link: feedItem.link,
         category: (feedItem.categories || []).map((category) => {
           return {
-            name: category,
+            name: escapeTextForXml(category),
           };
         }),
         author:
           feedItem.creator && typeof feedItem.creator === 'string'
             ? [
                 {
-                  name: feedItem.creator,
+                  name: escapeTextForXml(feedItem.creator),
                 },
               ]
             : undefined,
@@ -108,8 +118,8 @@ export class FeedGenerator {
             name: '_custom',
             objects: {
               hatenaCount: allFeedItemHatenaCountMap.get(feedItem.link) || 0,
-              originalTitle: feedItem.title,
-              blogTitle: feedItem.blogTitle,
+              originalTitle: escapeTextForXml(feedItem.title ?? ''),
+              blogTitle: escapeTextForXml(feedItem.blogTitle),
               blogLink: feedItem.blogLink,
               blogLinkMd5Hash: textToMd5Hash(feedItem.blogLink),
               favicon: ogObject?.favicon,

diff --git a/src/feed/utils/feed-validator.ts b/src/feed/utils/feed-validator.ts
@@ -3,15 +3,12 @@ import { XMLValidator } from 'fast-xml-parser';
 import type { Feed } from 'feed';
 import libxmljs from 'libxmljs';
 import RssParser from 'rss-parser';
-import { logger } from './logger';
 
 /**
  * フィードのバリデーション
  */
 export class FeedValidator {
   public async assertFeed(feed: Feed): Promise<void> {
-    logger.info('[FeedValidator] フィードのバリデーション開始');
-
     // 一つでもimageがあればok
     let isImageFound = false;
     for (const item of feed.items) {
@@ -23,20 +20,19 @@ export class FeedValidator {
     if (!isImageFound) {
       throw new Error('フィードに画像情報が一つもありません');
     }
-
-    logger.info('[FeedValidator] フィードのバリデーション完了');
   }
 
   public async assertXmlFeed(label: string, feedXml: string): Promise<void> {
     const rssParser = new RssParser();
 
-    logger.info(`[FeedValidator] XMLフィードのバリデーション開始。 label: ${label}`);
-
     // rss-parser で変換してみてエラーが出ないか確認
     const [rssParserError] = await to(rssParser.parseString(feedXml));
     if (rssParserError) {
       throw new Error(
         `rss-parserによるフィードのバリデーションエラーです。 label: ${label}, error: ${rssParserError}}`,
+        {
+          cause: rssParserError,
+        },
       );
     }
 
@@ -45,6 +41,9 @@ export class FeedValidator {
     if (atomValidateResult !== true) {
       throw new Error(
         `fast-xml-parser XMLValidatorによるフィードのバリデーションエラーです。 label: ${label}, result: ${atomValidateResult}`,
+        {
+          cause: atomValidateResult,
+        },
       );
     }
 
@@ -53,10 +52,13 @@ export class FeedValidator {
       libxmljs.parseXml(feedXml);
     } catch (libxmljsError) {
       if (libxmljsError instanceof Error) {
-        throw new Error(`libxmljsによるフィードのバリデーションエラーです。 error: ${libxmljsError.message}`);
+        throw new Error(
+          `libxmljsによるフィードのバリデーションエラーです。 label: ${label}, error: ${libxmljsError.message}`,
+          {
+            cause: libxmljsError,
+          },
+        );
       }
     }
-
-    logger.info(`[FeedValidator] XMLフィードのバリデーション完了。 label: ${label}`);
   }
 }
diff --git a/src/resources/feed-info-list.ts b/src/resources/feed-info-list.ts
@@ -267,7 +267,7 @@ export const FEED_INFO_LIST: FeedInfo[] = createFeedInfoList([
   ['SmartHR', 'https://tech.smarthr.jp/feed'],
   ['SmartNews', 'https://developer.smartnews.com/blog/feed'],
   ['SmartNewsメディア担当チーム', 'https://www.mediatechnology.jp/feed'],
-  ['Snowlfake', 'https://zenn.dev/p/dataheroes/feed'],
+  ['Snowflake', 'https://zenn.dev/p/dataheroes/feed'],
   ['Social Databank', 'https://zenn.dev/p/sdb_blog/feed'],
   ['Speee', 'https://tech.speee.jp/feed'],
   ['Spiral.AI', 'https://zenn.dev/p/spiralai/feed'],