From 23b92cee63e23b4cb69a056c6950d1af91baf9c7 Mon Sep 17 00:00:00 2001 From: Kyle King Date: Mon, 30 Jan 2023 22:36:12 -0500 Subject: [PATCH] feat: first pass at the package --- .gitignore | 4 +- docs/README.md | 25 ++++++----- poetry.lock | 10 ++--- pyproject.toml | 47 ++++++++++++--------- tail_jsonl/__init__.py | 6 +-- tail_jsonl/_private/__init__.py | 0 tail_jsonl/_private/config.py | 62 +++++++++++++++++++++++++++ tail_jsonl/_private/core.py | 75 +++++++++++++++++++++++++++++++++ tail_jsonl/main.py | 19 +++++++++ tail_jsonl/tmp.jsonl | 4 ++ 10 files changed, 209 insertions(+), 43 deletions(-) create mode 100644 tail_jsonl/_private/__init__.py create mode 100644 tail_jsonl/_private/config.py create mode 100644 tail_jsonl/_private/core.py create mode 100644 tail_jsonl/main.py create mode 100644 tail_jsonl/tmp.jsonl diff --git a/.gitignore b/.gitignore index e724124..9978ce5 100644 --- a/.gitignore +++ b/.gitignore @@ -144,7 +144,7 @@ dmypy.json .vscode/* *.csv *.doc* -*.jsonl +# *.jsonl *.lnk *.log *.pdf @@ -203,3 +203,5 @@ coverage.json # ---------------------------------------------------------------------------------------------------------------------- # Custom Rules + +.logs/*.jsonl diff --git a/docs/README.md b/docs/README.md index ddba34e..636af20 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,25 +2,26 @@ Tail JSONL Logs -## Installation +## Background -1. `poetry add ` +I wanted to find a tool that could: -1. ... +1. Convert a stream of JSONL logs into a readable `logfmt`-like output with minimal configuration +1. Show exceptions on their own line - ```sh - import +I investigated a lot of alternatives such as: [humanlog](https://github.com/humanlogio/humanlog), [lnav](https://docs.lnav.org/en/latest/formats.html#), [goaccess](https://goaccess.io/get-started), [angle-grinder](https://github.com/rcoh/angle-grinder#rendering), [jq](https://github.com/stedolan/jq), [textualog](https://github.com/rhuygen/textualog), etc. but None had the exception formatting I wanted. - # < TODO: Add example code here > - ``` +## Installation -1. ... +```sh +pipx install tail-jsonl +``` ## Usage - - -For more example code, see the [scripts] directory or the [tests]. +```sh +echo '{"message": "message", "timestamp": "2023-01-01T01:01:01.0123456Z", "level": "debug", "data": true, "more-data": [null, true, -123.123]}' | tail-jsonl +``` ## Project Status @@ -54,6 +55,4 @@ If you have any security issue to report, please contact the project maintainers [contributor-covenant]: https://www.contributor-covenant.org [developer_guide]: ./docs/DEVELOPER_GUIDE.md [license]: https://github.com/kyleking/tail-jsonl/LICENSE -[scripts]: https://github.com/kyleking/tail-jsonl/scripts [style_guide]: ./docs/STYLE_GUIDE.md -[tests]: https://github.com/kyleking/tail-jsonl/tests diff --git a/poetry.lock b/poetry.lock index 0bc01a7..ef7dcfe 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2216,7 +2216,7 @@ testing = ["coverage", "pyyaml"] name = "markdown-it-py" version = "2.1.0" description = "Python port of markdown-it. Markdown parsing, done right!" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2313,7 +2313,7 @@ files = [ name = "mdurl" version = "0.1.2" description = "Markdown URL utilities" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -3146,7 +3146,7 @@ files = [ name = "pygments" version = "2.14.0" description = "Pygments is a syntax highlighting package written in Python." -category = "dev" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -3723,7 +3723,7 @@ types-setuptools = ">=57.0.0" name = "rich" version = "13.3.1" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" -category = "dev" +category = "main" optional = false python-versions = ">=3.7.0" files = [ @@ -4568,4 +4568,4 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [metadata] lock-version = "2.0" python-versions = "^3.8.12" -content-hash = "2cfc5284e49d7ee9827410ac4c8dda22fe55ed114426029a7ee21b727ff08c17" +content-hash = "e175a546ab6932a4cb1ae2f96245ec109c3d6c8cd033df6aefbbc2fb3231de24" diff --git a/pyproject.toml b/pyproject.toml index 12c8d71..50c7bd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,34 +23,39 @@ line_length = 120 multi_line_output = 5 [tool.poetry] -name = "tail_jsonl" -version = "0.0.1" -description = "Tail JSONL Logs" -license = "MIT" authors = ["Kyle King "] -maintainers = [] -repository = "https://github.com/kyleking/tail-jsonl" +classifiers = [ + "Development Status :: 1 - Planning", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9" +] # https://pypi.org/classifiers/ +description = "Tail JSONL Logs" documentation = "https://tail-jsonl.kyleking.me" -readme = "docs/README.md" include = ["LICENSE.md"] keywords = [] -classifiers = [ - "Development Status :: 1 - Planning", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", -] # https://pypi.org/classifiers/ +license = "MIT" +maintainers = [] +name = "tail_jsonl" +readme = "docs/README.md" +repository = "https://github.com/kyleking/tail-jsonl" +version = "0.0.1" -[tool.poetry.urls] -"Bug Tracker" = "https://github.com/kyleking/tail-jsonl/issues" -"Changelog" = "https://github.com/kyleking/tail-jsonl/blob/main/docs/docs/CHANGELOG.md" + +[tool.poetry.scripts] +tail-jsonl = "tail_jsonl:main" [tool.poetry.dependencies] -python = "^3.8.12" calcipy = ">=0.21.3" +python = "^3.8.12" +rich = ">=13.3.1" [tool.poetry.dev-dependencies] -calcipy = { version = "*", extras = ["dev", "lint", "test"] } +calcipy = {extras = ["dev", "lint", "test"], version = "*"} + +[tool.poetry.urls] +"Bug Tracker" = "https://github.com/kyleking/tail-jsonl/issues" +"Changelog" = "https://github.com/kyleking/tail-jsonl/blob/main/docs/docs/CHANGELOG.md" diff --git a/tail_jsonl/__init__.py b/tail_jsonl/__init__.py index 0bc845f..7655aae 100644 --- a/tail_jsonl/__init__.py +++ b/tail_jsonl/__init__.py @@ -1,10 +1,10 @@ """tail_jsonl.""" -from loguru import logger +from loguru import logger # noqa: F401 __version__ = '0.0.1' __pkg_name__ = 'tail_jsonl' -logger.disable(__pkg_name__) - # ====== Above is the recommended code from calcipy_template and may be updated on new releases ====== + +from .main import main # noqa: F401 diff --git a/tail_jsonl/_private/__init__.py b/tail_jsonl/_private/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tail_jsonl/_private/config.py b/tail_jsonl/_private/config.py new file mode 100644 index 0000000..d0ce52e --- /dev/null +++ b/tail_jsonl/_private/config.py @@ -0,0 +1,62 @@ +"""Configuration.""" + +from functools import cached_property + +from beartype import beartype +from pydantic import BaseModel, Field + + +class Styles(BaseModel): + """Styles configuration. + + Refer to https://rich.readthedocs.io/en/latest/style.html for available style + + Inspired by: https://github.com/Delgan/loguru/blob/07f94f3c8373733119f85aa8b9ca05ace3325a4b/loguru/_defaults.py#L31-L73 + + And: https://github.com/hynek/structlog/blob/bcfc7f9e60640c150bffbdaeed6328e582f93d1e/src/structlog/dev.py#L126-L141 + + """ + + timestamp: str = 'dim grey' + message: str = '' + + level_error: str = 'bold red' + level_warn: str = 'bold yellow' + level_info: str = 'bold green' + level_debug: str = 'bold blue' + level_fallback: str = '' + + key: str = 'green' + value: str = '' + + @cached_property + def _level_lookup(self) -> dict[str, str]: + return { + 'ERROR': self.level_error, + 'WARNING': self.level_warn, + 'WARN': self.level_warn, + 'INFO': self.level_info, + 'DEBUG': self.level_debug, + } + + @beartype + def get_level_style(self, level: str) -> str: + """Return the right style for the specified level.""" + return self._level_lookup.get(level.upper(), self.level_fallback) + + +class Keys(BaseModel): + """Special Keys.""" + + timestamp: list[str] = Field(default_factory=lambda: ['timestamp']) + level: list[str] = Field(default_factory=lambda: ['level']) + message: list[str] = Field(default_factory=lambda: ['event', 'message']) + + on_own_line: list[str] = Field(default_factory=lambda: ['exception']) + + +class Config(BaseModel): + """`tail-jsonl` config.""" + + styles: Styles = Field(default_factory=Styles) + keys: Keys = Field(default_factory=Keys) diff --git a/tail_jsonl/_private/core.py b/tail_jsonl/_private/core.py new file mode 100644 index 0000000..76b123b --- /dev/null +++ b/tail_jsonl/_private/core.py @@ -0,0 +1,75 @@ +"""Core print logic.""" + +import json +from copy import copy +from typing import Any + +from beartype import beartype +from loguru import logger +from pydantic import BaseModel +from rich.console import Console +from rich.text import Text + +from .config import Config + + +@beartype +def pop_key(data: dict, keys: list[str], fallback: str) -> Any: + """Recursively pop whichever key matches first or default to the fallback.""" + try: + key = keys.pop(0) + return data.pop(key, None) or pop_key(data, keys, fallback) + except IndexError: + return fallback + + +class Record(BaseModel): + """Record Model.""" + + timestamp: str + level: str + message: str + data: dict + + @classmethod + def from_line(cls, data: dict, config: Config) -> 'Record': + """Extract Record from jsonl.""" + return cls( + timestamp=pop_key(data, copy(config.keys.timestamp), ''), + level=pop_key(data, copy(config.keys.level), ''), + message=pop_key(data, copy(config.keys.message), ''), + data=data, + ) + + +@beartype +def print_record(line: str, console: Console, config: Config) -> None: + """Format and print the record.""" + try: + record = Record.from_line(json.loads(line), config=config) + except Exception: + logger.exception('Error in tail-json to parse line', line=line) + console.print('') # Line break + return + + text = Text(tab_size=4) # FIXME: Why isn't this indenting what is wrapped? + text.append(f'{record.timestamp: <28}', style=config.styles.timestamp) + text.append(f' {record.level: <7}', style=config.styles.get_level_style(record.level)) + text.append(f' {record.message: <20}', style=config.styles.message) + + full_lines = [] + for key in config.keys.on_own_line: + line = record.data.pop(key, None) + if line: + full_lines.append((key, line)) + + for key, value in record.data.items(): + text.append(f' {key}:', style=config.styles.key) + text.append(f' {str(value): <10}', style=config.styles.value) + + console.print(text) + for key, line in full_lines: + new_text = Text() + new_text.append(f' ∟ {key}', style='bold green') + new_text.append(f': {line}') + console.print(new_text) diff --git a/tail_jsonl/main.py b/tail_jsonl/main.py new file mode 100644 index 0000000..1f241d0 --- /dev/null +++ b/tail_jsonl/main.py @@ -0,0 +1,19 @@ +"""Public CLI interface.""" + +import fileinput + +from beartype import beartype +from rich.console import Console + +from ._private.config import Config +from ._private.core import print_record + + +@beartype +def main() -> None: + """CLI Entrypoint.""" + console = Console() + # PLANNED: Support configuration with argparse or environment variable + config = Config() + for line in fileinput.input(): + print_record(line, console, config) diff --git a/tail_jsonl/tmp.jsonl b/tail_jsonl/tmp.jsonl new file mode 100644 index 0000000..fae37ca --- /dev/null +++ b/tail_jsonl/tmp.jsonl @@ -0,0 +1,4 @@ +{"jsonrpc": "2.0", "method": "sum", "params": [ null, 1, 2, 4, false, true], "id": "1"} +{"jsonrpc": "2.0", "method": "notify_hello", "params": [7]} +{"jsonrpc": "2.0", "method": "subtract", "params": [42, 23], "id": "2"} +{"event": "Testing", "user_id": "UUID('f86f89aa-a260-431a-a3a6-7b37a695e012'), UUID('f86f89aa-a260-431a-a3a6-7b37a695e012'), UUID('f86f89aa-a260-431a-a3a6-7b37a695e012'), UUID('f86f89aa-a260-431a-a3a6-7b37a695e012'), UUID('f86f89aa-a260-431a-a3a6-7b37a695e012'), UUID('f86f89aa-a260-431a-a3a6-7b37a695e012')", "something": 123, "level": "debug", "timestamp": "2022-12-22T16:52:28.380253Z", "func_name": "", "filename": "test.py", "lineno": 15, "exception": "line 1\n\t\tline 2 .... \n\t...?"}