From 02a2ac8975f6326234cb4a9ac9b972e8f682df1a Mon Sep 17 00:00:00 2001 From: pscott <30843220+pscott@users.noreply.github.com> Date: Sat, 23 Mar 2024 15:44:02 +0100 Subject: [PATCH] feat: add AI summary (#240) * feat: add AI summary * fix lint errors * chore: remove extra console logs * chore: fix order and duplicate import * refactor: code cleaning and refactoring * chore: add missing env key to test env --------- Co-authored-by: Wan Qi Chen <495709+wa0x6e@users.noreply.github.com> --- package.json | 3 +- src/api.ts | 23 ++++++++ src/helpers/snapshot.ts | 8 +++ src/lib/ai/summary.ts | 60 ++++++++++++++++++++ test/.env.test | 1 + yarn.lock | 122 +++++++++++++++++++++++++++++++++++++++- 6 files changed, 214 insertions(+), 3 deletions(-) create mode 100644 src/lib/ai/summary.ts diff --git a/package.json b/package.json index 179de2a..52347bb 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,8 @@ "graphql": "^16.7.1", "multiformats": "^9", "mysql": "^2.18.1", - "node-fetch": "^2.7.0" + "node-fetch": "^2.7.0", + "openai": "^4.29.2" }, "devDependencies": { "@snapshot-labs/eslint-config": "^0.1.0-beta.15", diff --git a/src/api.ts b/src/api.ts index 7bc184f..647ef0f 100644 --- a/src/api.ts +++ b/src/api.ts @@ -7,6 +7,7 @@ import mintPayload from './lib/nftClaimer/mint'; import deployPayload from './lib/nftClaimer/deploy'; import { queue, getProgress } from './lib/queue'; import { snapshotFee } from './lib/nftClaimer/utils'; +import AISummary from './lib/ai/summary'; const router = express.Router(); @@ -37,6 +38,28 @@ router.all('/votes/:id', async (req, res) => { } }); +router.post('/ai/summary/:id', async (req, res) => { + const { id } = req.params; + const aiSummary = new AISummary(id, storageEngine(process.env.AI_SUMMARY_SUBDIR)); + + try { + const cachedSummary = await aiSummary.getCache(); + + let summary = ''; + + if (!cachedSummary) { + summary = (await aiSummary.createCache()).toString(); + } else { + summary = cachedSummary.toString(); + } + + return rpcSuccess(res.status(200), summary, id); + } catch (e) { + capture(e); + return rpcError(res, 'INTERNAL_ERROR', id); + } +}); + router.get('/moderation', async (req, res) => { const { list } = req.query; diff --git a/src/helpers/snapshot.ts b/src/helpers/snapshot.ts index 8fdced6..c7c1a0b 100644 --- a/src/helpers/snapshot.ts +++ b/src/helpers/snapshot.ts @@ -9,6 +9,9 @@ export type Proposal = { space: Space; votes: number; author: string; + title: string; + body: string; + discussion: string; }; export type Vote = { @@ -23,6 +26,7 @@ export type Vote = { export type Space = { id: string; network: string; + name: string; }; const httpLink = createHttpLink({ @@ -65,10 +69,14 @@ const PROPOSAL_QUERY = gql` state choices votes + title + body + discussion author space { id network + name } } } diff --git a/src/lib/ai/summary.ts b/src/lib/ai/summary.ts new file mode 100644 index 0000000..57fff1d --- /dev/null +++ b/src/lib/ai/summary.ts @@ -0,0 +1,60 @@ +import OpenAI from 'openai'; +import { capture } from '@snapshot-labs/snapshot-sentry'; +import { fetchProposal, Proposal } from '../../helpers/snapshot'; +import { IStorage } from '../storage/types'; +import Cache from '../cache'; + +const openai = new OpenAI({ apiKey: process.env.apiKey }); + +class AISummary extends Cache { + proposal?: Proposal | null; + + constructor(id: string, storage: IStorage) { + super(id, storage); + this.filename = `snapshot-proposal-ai-summary-${this.id}.txt`; + } + + async isCacheable() { + this.proposal = await fetchProposal(this.id); + + if (!this.proposal) { + return Promise.reject('RECORD_NOT_FOUND'); + } + + return true; + } + + getContent = async () => { + this.isCacheable(); + + try { + const { body, title, space } = this.proposal!; + + const completion = await openai.chat.completions.create({ + messages: [ + { + role: 'system', + content: `The following is a governance proposal of ${space.name}. Generate me a summary in 300 characters max. Here's the title of the proposal: '${title}' . Here's the content of the proposal: '${body}'` + } + ], + model: 'gpt-4-turbo-preview' + }); + + if (completion.choices.length === 0) { + throw new Error('No completion in response'); + } + const content = completion.choices[0].message.content; + + if (!content) { + throw new Error('No content in response'); + } + + return content; + } catch (e: any) { + capture(e); + throw e; + } + }; +} + +export default AISummary; diff --git a/test/.env.test b/test/.env.test index f81ed92..cd5f969 100644 --- a/test/.env.test +++ b/test/.env.test @@ -9,6 +9,7 @@ NFT_CLAIMER_DEPLOY_IMPLEMENTATION_ADDRESS=0x33505720a7921d23E6b02EB69623Ed6A008C NFT_CLAIMER_DEPLOY_INITIALIZE_SELECTOR=0x977b0efb NFT_CLAIMER_SUBGRAPH_URL='https://api.studio.thegraph.com/query/48277/nft-subgraph-goerli/version/latest' STORAGE_ENGINE=file +OPENAI_API_KEY='' # your own test AWS for credentials if you wish to test with AWS # AWS_ACCESS_KEY_ID= # AWS_REGION= diff --git a/yarn.lock b/yarn.lock index 1dc3034..1ded73d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2335,6 +2335,13 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-20.4.8.tgz#b5dda19adaa473a9bf0ab5cbd8f30ec7d43f5c85" integrity sha512-0mHckf6D2DiIAzh8fM8f3HQCvMKDpK94YQ0DSVkfWTG9BZleYIWudw9cJxX8oCk9bM+vAkDyujDV6dmKHbvQpg== +"@types/node@^18.11.18": + version "18.19.26" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.26.tgz#18991279d0a0e53675285e8cf4a0823766349729" + integrity sha512-+wiMJsIwLOYCvUqSdKTrfkS8mpTp+MPINe6+Np4TAGFWWRWiBQ5kSq9nZGCSPkzx9mvT+uEukzpX4MOSCydcvw== + dependencies: + undici-types "~5.26.4" + "@types/qs@*": version "6.9.7" resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb" @@ -2589,6 +2596,13 @@ abbrev@1: resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.1.1.tgz#f8f2c887ad10bf67f634f005b6987fed3179aac8" integrity sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q== +abort-controller@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" + integrity sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg== + dependencies: + event-target-shim "^5.0.0" + accepts@~1.3.5, accepts@~1.3.8: version "1.3.8" resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.8.tgz#0bf0be125b67014adcb0b0921e62db7bffe16b2e" @@ -2629,6 +2643,13 @@ agent-base@6: dependencies: debug "4" +agentkeepalive@^4.2.1: + version "4.5.0" + resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.5.0.tgz#2673ad1389b3c418c5a20c5d7364f93ca04be923" + integrity sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew== + dependencies: + humanize-ms "^1.2.1" + ajv-errors@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/ajv-errors/-/ajv-errors-3.0.0.tgz#e54f299f3a3d30fe144161e5f0d8d51196c527bc" @@ -2883,6 +2904,11 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== +base-64@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/base-64/-/base-64-0.1.0.tgz#780a99c84e7d600260361511c4877613bf24f6bb" + integrity sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA== + bech32@1.1.4: version "1.1.4" resolved "https://registry.yarnpkg.com/bech32/-/bech32-1.1.4.tgz#e38c9f37bf179b8eb16ae3a772b40c356d4832e9" @@ -3087,6 +3113,11 @@ char-regex@^1.0.2: resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf" integrity sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw== +charenc@0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/charenc/-/charenc-0.0.2.tgz#c0a1d2f3a7092e03774bfa83f14c0fc5790a8667" + integrity sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA== + check-more-types@2.24.0: version "2.24.0" resolved "https://registry.yarnpkg.com/check-more-types/-/check-more-types-2.24.0.tgz#1420ffb10fd444dcfc79b43891bbfffd32a84600" @@ -3295,6 +3326,11 @@ cross-spawn@^7.0.2, cross-spawn@^7.0.3: shebang-command "^2.0.0" which "^2.0.1" +crypt@0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/crypt/-/crypt-0.0.2.tgz#88d7ff7ec0dfb86f713dc87bbb42d044d3e6c41b" + integrity sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow== + debug@2.6.9: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" @@ -3410,6 +3446,14 @@ diff@^4.0.1: resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== +digest-fetch@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/digest-fetch/-/digest-fetch-1.3.0.tgz#898e69264d00012a23cf26e8a3e40320143fc661" + integrity sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA== + dependencies: + base-64 "^0.1.0" + md5 "^2.3.0" + dir-glob@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f" @@ -3768,6 +3812,11 @@ event-stream@=3.3.4: stream-combiner "~0.0.4" through "~2.3.1" +event-target-shim@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/event-target-shim/-/event-target-shim-5.0.1.tgz#5d4d3ebdf9583d63a5333ce2deb7480ab2b05789" + integrity sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ== + execa@5.1.1, execa@^5.0.0: version "5.1.1" resolved "https://registry.yarnpkg.com/execa/-/execa-5.1.1.tgz#f80ad9cbf4298f7bd1d4c9555c21e93741c411dd" @@ -3996,6 +4045,11 @@ for-each@^0.3.3: dependencies: is-callable "^1.1.3" +form-data-encoder@1.7.2: + version "1.7.2" + resolved "https://registry.yarnpkg.com/form-data-encoder/-/form-data-encoder-1.7.2.tgz#1f1ae3dccf58ed4690b86d87e4f57c654fbab040" + integrity sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A== + form-data@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.1.tgz#ebd53791b78356a99af9a300d4282c4d5eb9755f" @@ -4014,6 +4068,14 @@ form-data@^4.0.0: combined-stream "^1.0.8" mime-types "^2.1.12" +formdata-node@^4.3.2: + version "4.4.1" + resolved "https://registry.yarnpkg.com/formdata-node/-/formdata-node-4.4.1.tgz#23f6a5cb9cb55315912cbec4ff7b0f59bbd191e2" + integrity sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ== + dependencies: + node-domexception "1.0.0" + web-streams-polyfill "4.0.0-beta.3" + formidable@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/formidable/-/formidable-2.1.2.tgz#fa973a2bec150e4ce7cac15589d7a25fc30ebd89" @@ -4323,6 +4385,13 @@ human-signals@^4.3.0: resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-4.3.1.tgz#ab7f811e851fca97ffbd2c1fe9a958964de321b2" integrity sha512-nZXjEF2nbo7lIw3mgYjItAfgQXog3OjJogSbKa2CQIIvSGWcKgeJnQlNXip6NglNzYH45nSRiEVimMvYL8DDqQ== +humanize-ms@^1.2.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/humanize-ms/-/humanize-ms-1.2.1.tgz#c46e3159a293f6b896da29316d8b6fe8bb79bbed" + integrity sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ== + dependencies: + ms "^2.0.0" + iconv-lite@0.4.24: version "0.4.24" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" @@ -4424,6 +4493,11 @@ is-boolean-object@^1.1.0: call-bind "^1.0.2" has-tostringtag "^1.0.0" +is-buffer@~1.1.6: + version "1.1.6" + resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" + integrity sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w== + is-callable@^1.1.3, is-callable@^1.1.4, is-callable@^1.2.7: version "1.2.7" resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.7.tgz#3bc2a85ea742d9e36205dcacdd72ca1fdc51b055" @@ -5238,6 +5312,15 @@ map-stream@~0.1.0: resolved "https://registry.yarnpkg.com/map-stream/-/map-stream-0.1.0.tgz#e56aa94c4c8055a16404a0674b78f215f7c8e194" integrity sha512-CkYQrPYZfWnu/DAmVCpTSX/xHpKZ80eKh2lAkyA6AJTef6bW+6JpbQZN5rofum7da+SyN1bi5ctTm+lTfcCW3g== +md5@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/md5/-/md5-2.3.0.tgz#c3da9a6aae3a30b46b7b0c349b87b110dc3bda4f" + integrity sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g== + dependencies: + charenc "0.0.2" + crypt "0.0.2" + is-buffer "~1.1.6" + media-typer@0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" @@ -5340,7 +5423,7 @@ ms@2.1.2: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009" integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w== -ms@2.1.3, ms@^2.1.1: +ms@2.1.3, ms@^2.0.0, ms@^2.1.1: version "2.1.3" resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== @@ -5370,7 +5453,12 @@ negotiator@0.6.3: resolved "https://registry.yarnpkg.com/negotiator/-/negotiator-0.6.3.tgz#58e323a72fedc0d6f9cd4d31fe49f51479590ccd" integrity sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg== -node-fetch@^2.6.12, node-fetch@^2.7.0: +node-domexception@1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" + integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== + +node-fetch@^2.6.12, node-fetch@^2.6.7, node-fetch@^2.7.0: version "2.7.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== @@ -5533,6 +5621,21 @@ open@^9.1.0: is-inside-container "^1.0.0" is-wsl "^2.2.0" +openai@^4.29.2: + version "4.29.2" + resolved "https://registry.yarnpkg.com/openai/-/openai-4.29.2.tgz#45e83cb49dbd052626637b267c749785a24f411c" + integrity sha512-cPkT6zjEcE4qU5OW/SoDDuXEsdOLrXlAORhzmaguj5xZSPlgKvLhi27sFWhLKj07Y6WKNWxcwIbzm512FzTBNQ== + dependencies: + "@types/node" "^18.11.18" + "@types/node-fetch" "^2.6.4" + abort-controller "^3.0.0" + agentkeepalive "^4.2.1" + digest-fetch "^1.3.0" + form-data-encoder "1.7.2" + formdata-node "^4.3.2" + node-fetch "^2.6.7" + web-streams-polyfill "^3.2.1" + optimism@^0.18.0: version "0.18.0" resolved "https://registry.yarnpkg.com/optimism/-/optimism-0.18.0.tgz#e7bb38b24715f3fdad8a9a7fc18e999144bbfa63" @@ -6553,6 +6656,11 @@ undefsafe@^2.0.5: resolved "https://registry.yarnpkg.com/undefsafe/-/undefsafe-2.0.5.tgz#38733b9327bdcd226db889fb723a6efd162e6e2c" integrity sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA== +undici-types@~5.26.4: + version "5.26.5" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" + integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== + unpipe@1.0.0, unpipe@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" @@ -6635,6 +6743,16 @@ walker@^1.0.8: dependencies: makeerror "1.0.12" +web-streams-polyfill@4.0.0-beta.3: + version "4.0.0-beta.3" + resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz#2898486b74f5156095e473efe989dcf185047a38" + integrity sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug== + +web-streams-polyfill@^3.2.1: + version "3.3.3" + resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz#2073b91a2fdb1fbfbd401e7de0ac9f8214cecb4b" + integrity sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw== + webidl-conversions@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"