initial commit of plugin

trashhalo · Jun 2, 2022 · d88dd99 · d88dd99
1 parent b6fc384
commit d88dd99
Show file tree

Hide file tree

Showing 20 changed files with 723 additions and 0 deletions.
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -0,0 +1,67 @@
+name: Build plugin
+
+on:
+  push:
+    # Sequence of patterns matched against refs/tags
+    tags:
+      - "*" # Push events to matching any tag format, i.e. 1.0, 20.15.10
+
+env:
+  PLUGIN_NAME: logseq-summarizer
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js
+        uses: actions/setup-node@v1
+        with:
+          node-version: "14.x" # You might need to adjust this value to your own version
+      - name: Build
+        id: build
+        run: |
+          npm install -g yarn
+          yarn
+          yarn workspace @summarizer/worker build
+          yarn workspace @summarizer/logseq build
+          mkdir ${{ env.PLUGIN_NAME }}
+          cp README.md logseq/package.json logseq/icon.png logseq/demo.gif ${{ env.PLUGIN_NAME }}
+          mv logseq/dist ${{ env.PLUGIN_NAME }}
+          zip -r ${{ env.PLUGIN_NAME }}.zip ${{ env.PLUGIN_NAME }}
+          ls
+          echo "::set-output name=tag_name::$(git tag --sort version:refname | tail -n 1)"
+
+      - name: Create Release
+        uses: ncipollo/release-action@v1
+        id: create_release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          VERSION: ${{ github.ref }}
+        with:
+          allowUpdates: true
+          draft: false
+          prerelease: false
+
+      - name: Upload zip file
+        id: upload_zip
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./${{ env.PLUGIN_NAME }}.zip
+          asset_name: ${{ env.PLUGIN_NAME }}-${{ steps.build.outputs.tag_name }}.zip
+          asset_content_type: application/zip
+
+      - name: Upload package.json
+        id: upload_metadata
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./logseq/package.json
+          asset_name: package.json
+          asset_content_type: application/json
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,3 @@
+node_modules/
+yarn-error.log
+.DS_store
diff --git a/README.md b/README.md
@@ -0,0 +1,22 @@
+# Logseq Summarizer
+
+Uses [sumy](https://pypi.org/project/sumy/) to summarize long text in logseq
+
+![demo](./logseq/demo.gif)
+
+## Installation
+
+- Download a released version assets from Github.
+- Unzip it.
+- Click Load unpacked plugin, and select destination directory to the unzipped folder.
+
+## Development
+
+1. yarn
+2. yarn workspace @summarizer/worker build
+3. yarn workspace @summarizer/logseq build
+4. Load the unpacked plugin
+
+## Icon
+
+[Data compression icons created by Eucalyp - Flaticon](https://www.flaticon.com/free-icons/data-compression)
diff --git a/logseq/.gitignore b/logseq/.gitignore
@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
diff --git a/logseq/demo.gif b/logseq/demo.gif
diff --git a/logseq/dist/assets/index.8e3ea1aa.js b/logseq/dist/assets/index.8e3ea1aa.js
diff --git a/logseq/dist/index.html b/logseq/dist/index.html
@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>App</title>
+    <script type="module" crossorigin src="./assets/index.8e3ea1aa.js"></script>
+  </head>
+  <body>
+    <div id="app"></div>
+
+  </body>
+</html>
diff --git a/logseq/icon.png b/logseq/icon.png
diff --git a/logseq/index.html b/logseq/index.html
@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>App</title>
+  </head>
+  <body>
+    <div id="app"></div>
+    <script type="module" src="/index.js"></script>
+  </body>
+</html>
diff --git a/logseq/index.js b/logseq/index.js
@@ -0,0 +1,65 @@
+import "@logseq/libs";
+import createWorker from "@summarizer/worker";
+
+const showError = (err) => {
+  logseq.App.showMsg(err.toString(), "error");
+};
+
+const settingsSchema = [
+  {
+    key: "language",
+    type: "string",
+    title: "Language to use for summaries",
+    description:
+      "What language do you want to default to when summarizing text?",
+    default: "english",
+  },
+  {
+    key: "numberOfSentences",
+    type: "string",
+    title: "Number of sentences",
+    description:
+      "How many sentences do you want to generate when you summarize content?",
+    default: 3,
+  },
+];
+
+let worker;
+
+function main() {
+  worker = createWorker();
+
+  logseq.Editor.registerSlashCommand("Summary", async () => {
+    try {
+      console.log("settings", logseq.settings);
+      const { language, numberOfSentences } = logseq.settings;
+      const { content, uuid } = await logseq.Editor.getCurrentBlock();
+      const summary = await worker.postMessage({
+        language,
+        count: numberOfSentences,
+        text: content,
+      });
+
+      await logseq.Editor.updateBlock(uuid, summary[0]);
+      await logseq.Editor.insertBatchBlock(
+        uuid,
+        summary.slice(1).map((c) => {
+          return { content: c };
+        }),
+        { sibling: true }
+      );
+    } catch (err) {
+      showError(err);
+    }
+  });
+}
+
+logseq.beforeunload(() => {
+  if (worker) {
+    worker._worker.terminate();
+    worker = null;
+  }
+});
+
+logseq.useSettingsSchema(settingsSchema);
+logseq.ready(main).catch(showError);
diff --git a/logseq/package.json b/logseq/package.json
@@ -0,0 +1,21 @@
+{
+  "name": "@summarizer/logseq",
+  "version": "0.0.0",
+  "main": "dist/index.html",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "preview": "vite preview"
+  },
+  "devDependencies": {
+    "vite": "^2.9.9"
+  },
+  "dependencies": {
+    "@logseq/libs": "^0.0.6",
+    "@summarizer/worker": "^0.0.0"
+  },
+  "logseq": {
+    "id": "_trashhalo-logseq-summarizer",
+    "icon": "./logo.png"
+  }
+}
diff --git a/logseq/vite.config.js b/logseq/vite.config.js
@@ -0,0 +1,8 @@
+/**
+ * @type {import('vite').UserConfig}
+ */
+const config = {
+  base: "./",
+};
+
+export default config;
diff --git a/package.json b/package.json
@@ -0,0 +1,10 @@
+{
+  "name": "@summarizer/monorepo",
+  "version": "0.0.0",
+  "private": true,
+  "license": "MIT",
+  "workspaces": [
+    "worker",
+    "logseq"
+  ]
+}
diff --git a/worker/.gitignore b/worker/.gitignore
@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
diff --git a/worker/lib.js b/worker/lib.js
@@ -0,0 +1,3 @@
+import PromiseWorker from "promise-worker";
+import InlineWorker from "./worker.js?worker&inline";
+export default () => new PromiseWorker(new InlineWorker());
diff --git a/worker/package.json b/worker/package.json
@@ -0,0 +1,16 @@
+{
+  "name": "@summarizer/worker",
+  "version": "0.0.0",
+  "main": "./dist/summarizer-worker.es.js",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build -d",
+    "preview": "vite preview"
+  },
+  "devDependencies": {
+    "vite": "^2.9.9"
+  },
+  "dependencies": {
+    "promise-worker": "^2.0.1"
+  }
+}
diff --git a/worker/summarize.py b/worker/summarize.py
@@ -0,0 +1,49 @@
+from __future__ import absolute_import
+from __future__ import division, print_function, unicode_literals
+from js import fetch
+import numpy as np
+from pathlib import Path
+import zipfile
+from pyodide import to_js
+
+# Snagged from https://github.com/pyodide/pyodide/issues/1798#issuecomment-968370996
+# Need to download the nltk dataset. Expects it to be in the file system. I believe this
+# is holding it in memory
+response = await fetch('https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/packages/tokenizers/punkt.zip')
+js_buffer = await response.arrayBuffer()
+py_buffer = js_buffer.to_py()
+stream = py_buffer.tobytes()
+d = Path("/nltk_data/tokenizers")
+d.mkdir(parents=True, exist_ok=True)
+Path('/nltk_data/tokenizers/punkt.zip').write_bytes(stream)
+zipfile.ZipFile('/nltk_data/tokenizers/punkt.zip').extractall(
+    path='/nltk_data/tokenizers/'
+)
+
+import micropip
+# micropip installs the transitive deps of sumy from pypi. there are 2 missing. I manually built
+# their wheels and uploaded them to a repo so I could install them at runtime. Can be found here
+# https://github.com/trashhalo/wheels
+await micropip.install(docopts_url)
+await micropip.install(breadability_url)
+await micropip.install('sumy')
+
+from sumy.parsers.plaintext import PlaintextParser
+from sumy.nlp.tokenizers import Tokenizer
+from sumy.summarizers.lsa import LsaSummarizer as Summarizer
+from sumy.nlp.stemmers import Stemmer
+from sumy.utils import get_stop_words
+
+def summarize(language, count, text):
+  parser = PlaintextParser.from_string(text, Tokenizer(language))
+  stemmer = Stemmer(language)
+
+  summarizer = Summarizer(stemmer)
+  summarizer.stop_words = get_stop_words(language)
+
+  result = []
+  for sentence in summarizer(parser.document, count):
+    result.append(sentence.__str__())
+  return to_js(result)
+
+summarize
diff --git a/worker/vite.config.js b/worker/vite.config.js
@@ -0,0 +1,18 @@
+const path = require("path");
+/**
+ * @type {import('vite').UserConfig}
+ */
+const config = {
+  build: {
+    lib: {
+      entry: path.resolve(__dirname, "lib.js"),
+      name: "@summarizer/worker",
+      fileName: (format) => `summarizer-worker.${format}.js`,
+    },
+  },
+  worker: {
+    format: "iife",
+  },
+};
+
+export default config;
diff --git a/worker/worker.js b/worker/worker.js
@@ -0,0 +1,26 @@
+importScripts("https://cdn.jsdelivr.net/pyodide/v0.20.0/full/pyodide.js");
+import registerPromiseWorker from "promise-worker/register";
+import summarize_py from "./summarize.py?raw";
+
+async function main() {
+  let pyodide = await loadPyodide();
+  await pyodide.loadPackage("nltk");
+  await pyodide.loadPackage("numpy");
+  await pyodide.loadPackage("micropip");
+  pyodide.globals.set(
+    "docopts_url",
+    "https://trashhalo.github.io/wheels/docopt-0.6.2-py2.py3-none-any.whl"
+  );
+  pyodide.globals.set(
+    "breadability_url",
+    "https://trashhalo.github.io/wheels/breadability-0.1.20-py2.py3-none-any.whl"
+  );
+  return pyodide.runPythonAsync(summarize_py);
+}
+
+const summarize_promise = main();
+
+registerPromiseWorker(async (message) => {
+  const summarize = await summarize_promise;
+  return summarize(message.language, message.count, message.text);
+});