diff --git a/.coveragerc b/.coveragerc index c0dcff0..41cd7cf 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,7 +1,11 @@ [run] branch = True source = wildq -omit = wildq/__main__.py +omit = + wildq/__main__.py + wildq/_wildq_version.py + tests/data/* + */site-packages/* [report] exclude_lines = diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..6aeb900 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,33 @@ +Hi there, + +Thank you for opening an issue. Please note that we try to keep the Wildq issue tracker reserved for bug reports and feature requests. + +### Wildq Version +Run `wildq --version` to show the version. If you are not running the latest version of Wildq, please upgrade because your issue may have already been fixed. + +### Affected FileType(s) +Please list the filetypes as a list, for example: +- json +- toml + +If this issue appears to affect multiple filetypes, it may be an issue with Wildq's core and not third-parties, so please mention this. + +### Debug Output +Please provider a link to a GitHub Gist containing the complete output. Please do NOT paste the debug output in the issue; just paste a link to the Gist. + +### Panic Output +If wildq produced a stacktrace, please provide a link to a GitHub Gist containing the output. + +### Expected Behavior +What should have happened? + +### Actual Behavior +What actually happened? + +### Steps to Reproduce +Please list the steps required to reproduce the issue, for example: +1. `wildq -i ...` + +### References +Are there any other GitHub issues (open or closed) or Pull Requests that should be linked here? For example: +- GH-1234 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..6a1d38f --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,84 @@ +name: Publish packages + +on: + push: + tags: + - 'v*' + +jobs: + create_release: + name: Create release + runs-on: ubuntu-latest + outputs: + id: ${{ steps.draft_release.outputs.id }} + html_url: ${{ steps.draft_release.outputs.html_url }} + upload_url: ${{ steps.draft_release.outputs.upload_url }} + steps: + - name: Create release + id: draft_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ github.ref }} + draft: true + prerelease: false + + assets: + name: Release packages + runs-on: ${{ matrix.os }} + needs: create_release + strategy: + fail-fast: false + matrix: + os: + - macos-latest + - ubuntu-latest + steps: + - name: Check out src from Git + uses: actions/checkout@v2 + + - name: Get history and tags for SCM versioning to work + run: | + git fetch --prune --unshallow + git fetch --depth=1 origin +refs/tags/*:refs/tags/* + + - name: Set up Python 3.6 + uses: actions/setup-python@v1 + with: + python-version: 3.6 + + - name: Build a binary wheel and a source tarball + run: | + python -m pip install -U setuptools pipenv wheel + make init sync + python setup.py --version + make build binary + zip --junk-paths wildq-${{runner.os}}-x86_64 dist/wq dist/wildq + rm -f dist/wq dist/wildq + + - name: Upload Release Asset + id: upload-release-asset + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.create_release.outputs.upload_url }} + asset_path: ./wildq-${{runner.os}}-x86_64.zip + asset_name: wildq-${{runner.os}}-x86_64.zip + asset_content_type: application/zip + + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@master + if: matrix.os == 'ubuntu-latest' + with: + password: ${{ secrets.GH_ACTIONS_WILD }} + + - name: Unset draft + if: matrix.os == 'ubuntu-latest' + uses: eregon/publish-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + release_id: ${{ needs.create_release.outputs.id }} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000..753dab6 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,33 @@ +name: Unit Tests +on: + push: + branches: + - '*' + pull_request: + branches: + - '*' + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [3.6, 3.7, 3.8] + os: [ubuntu-latest] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip wheel setuptools pipenv + make init + - name: Syntax + run: | + make syntax + - name: Test + run: | + make test diff --git a/.gitignore b/.gitignore index 04e3887..c7c0252 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,7 @@ var/ *.egg-info/ .installed.cfg *.egg +.eggs # PyInstaller # Usually these files are written by a python script from a template @@ -183,4 +184,10 @@ ignore/ .venv trash/ result.json -Pipfile* +.DS_Store +.vscode +pip-wheel-metadata/ +# wildq/_wildq_version.py +.idea +*.swp +Pipfile.lock diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index 801fcf5..0000000 --- a/.pylintrc +++ /dev/null @@ -1,336 +0,0 @@ -[MASTER] - -# Specify a configuration file. -#rcfile= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Profiled execution. -profile=no - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Pickle collected data for later comparisons. -persistent=yes - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# DEPRECATED -include-ids=no - -# DEPRECATED -symbols=no - - -[MESSAGES CONTROL] - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. See also the "--disable" option for examples. -#enable= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -#disable= - - -[REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=colorized - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no - -# Tells whether to display a full report or only the messages -reports=yes - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Add a comment according to your evaluation note. This is used by the global -# evaluation report (RP0004). -comment=no - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - - -[SIMILARITIES] - -# Minimum lines number of a similarity. -min-similarity-lines=4 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[TYPECHECK] - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis -ignored-modules= - -# List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamically set). -ignored-classes=SQLObject - -# When zope mode is activated, add a predefined set of Zope acquired attributes -# to generated-members. -zope=no - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E0201 when accessed. Python regular -# expressions are accepted. -generated-members=REQUEST,acl_users,aq_parent - - -[FORMAT] - -# Maximum number of characters on a single line. -max-line-length=80 - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - -# List of optional constructs for which whitespace checking is disabled -no-space-check=trailing-comma,dict-separator - -# Maximum number of lines in a module -max-module-lines=1000 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - - -[BASIC] - -# Required attributes for module, separated by a comma -required-attributes= - -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,apply,input,file - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# Regular expression matching correct function names -function-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for function names -function-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct variable names -variable-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for variable names -variable-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct constant names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression matching correct attribute names -attr-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for attribute names -attr-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct argument names -argument-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for argument names -argument-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression matching correct method names -method-rgx=[a-z_][a-z0-9_]{2,30}$ - -# Naming hint for method names -method-name-hint=[a-z_][a-z0-9_]{2,30}$ - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=__.*__ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO - - -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_$|dummy - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - - -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,TERMIOS,Bastion,rexec - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=5 - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.* - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of branch for function / method body -max-branches=12 - -# Maximum number of statements in function / method body -max-statements=50 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - - -[CLASSES] - -# List of interface methods to ignore, separated by a comma. This is used for -# instance to not check methods defines in Zope's Interface base class. -ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 36c4e59..0000000 --- a/.travis.yml +++ /dev/null @@ -1,26 +0,0 @@ -sudo: required -dist: xenial - -language: python - -python: - - '3.6' - - '3.7' - - '3.8' - -install: - - pip install tox-travis - -script: - - tox - - tox -e lint - -deploy: - provider: pypi - user: "__token__" - password: - secure: "zoklW1G25tsz1TCIXThHyUpQh6x72DpmrxW4dCPp4Nw1eNuPgtr3V374bo980OLupf+XKcoCAjVA6i6JP2dhrkLrw4vdbZo6+KERsrRczfhdvfXJ+rQ4gA0ODMMhXKISKmLSoB4QWCDk7z5af0mccHB5HvNOebnCTOH46F5TMoVvIHduu5I9VNy+8ZMvL5mjYtyok/+sryl5LfBgIwfcc4ZQiBX/zmj1iE37jwfG+e6NH1QWmy/MwHVmK58UkER6O5U9TV7rbbn/zyYw9Kov+Tmg8fFfJ2x8WytewEIfpF7XWuH5cMm/0t4q5o2OzBj9TP/aFCp5+JR5C7TgAboRb22aYD5HJLCE67r2uHiY0F0X3hBmacdI5gHoA72InVejHlArnXYEsrio1joTMw3uCuy5oVDMRTefku/kjLhE9q+Agx3lXhjnzvzPBiURDf7yWkoO7ede9XCTcDWWQk1WXZPHW3U+BbiucVJoLnfUiFdAX1xDVsUaSyYFjZsS1xhogA9Lh8S2IRuojkNuKG7uvhoI3pzyx7r50WDrDnvofRVZ8rz7txrNn3VhjUA4X/v8AfuM2q64Cy83eR7OzjPk5mNrbQ6U5TFUA6M12wnz+HwyQBnXWLFBo6xbIzDySPCb99ZdmZwvaIF8fJ3TXYFIkahURW8P3ZJK1caGS1gkkGo=" - on: - tags: true - python: 3.6 - distributions: "bdist_wheel" diff --git a/LICENSE b/LICENSE index 261eeb9..93aec43 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2020 Ahmet Demir Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e4c804e --- /dev/null +++ b/Makefile @@ -0,0 +1,49 @@ +PIPENV_CMD ?= pipenv run + +all: init fmt sync test syntax tests + +init: + pip show -q pipenv || pip install --user pipenv + pipenv lock --pre + pipenv install + pipenv install --dev + ${PIPENV_CMD} python setup.py --version + +fmt: + ${PIPENV_CMD} black . + +sync: + ${PIPENV_CMD} pipenv-setup sync + ${PIPENV_CMD} pipenv-setup sync --dev + ${PIPENV_CMD} python setup.py --version + +docs: + ${PIPENV_CMD} sphinx-build -b html docs docs/_build/html + +test: + ${PIPENV_CMD} coverage run -m unittest discover + ${PIPENV_CMD} coverage report -m + +syntax: + ${PIPENV_CMD} flake8 wildq --count --exit-zero --statistics + ${PIPENV_CMD} bandit -r wildq + +build: + ${PIPENV_CMD} python setup.py --version + cat wildq/_wildq_version.py + ${PIPENV_CMD} python setup.py bdist_wheel + +binary: + ${PIPENV_CMD} pyinstaller --clean --onefile --hidden-import=pkg_resources.py2_warn --name wildq wildq/__main__.py + chmod +x dist/wildq + cp dist/wildq dist/wq + ./tests/tests.sh + +pypi: + ${PIPENV_CMD} python setup.py register -r pypi + ${PIPENV_CMD} python setup.py sdist upload -r pypi + +clean: + rm -rf dist build .eggs wildq.egg-info + +.PHONY: all fmt init sync docs tests syntax build binary pypi clean diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..b3576b4 --- /dev/null +++ b/Pipfile @@ -0,0 +1,30 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] +bandit = "*" +black = "*" +coverage = "*" +flake8 = "*" +flake8-bugbear = "*" +pipenv = "*" +pipenv-setup = "*" +pre-commit = "*" +pyinstaller = "*" +setuptools_scm = "*" +Sphinx = "*" +sphinx-rtd-theme = "*" + +[packages] +pyyaml = "*" +toml = "*" +xmltodict = "*" +jq = "*" +pyhcl = "*" +click = "*" +pygments = "*" + +[requires] +python_version = "3.6" diff --git a/README.md b/README.md index 858abd3..cfb6adc 100644 --- a/README.md +++ b/README.md @@ -8,29 +8,110 @@ Purpose of this package is to provide a simple wrapper arround jq for different I'm tired of searching a package doing yaml jq, toml jq, ini jq etc. mainly used for scripting. This script uses: - * @mwilliamson [Python bindings](https://github.com/mwilliamson/jq.py) on top of @stedolan famous [jq](https://github.com/stedolan/jq/) lib -* @martinblech [xmldict](https://github.com/martinblech/xmltodict) to manage XML -* @uiri [toml](https://github.com/uiri/toml) to manage TOML -* @yaml [pyyaml](https://github.com/yaml/pyyaml) to manage YAML -* @virtuald [pyhcl](https://github.com/virtuald/pyhcl) to manage HCL -* for INI [ConfigParser](https://docs.python.org/3/library/configparser.html) is used. +* swiss knife for coloration [pygments](https://github.com/pygments/pygments) +* binary built with [pyinstaller](https://github.com/pyinstaller/pyinstaller) +* easy CLI with [click](https://github.com/pallets/click) +* for supported types sources, check table `Supported file types` # Installation +## Pip + ```sh pip install wildq ``` +## Binary + +A binary is also available for different platform, pick one (both of `wildq` and `wq` will be in the archive) + +### GNU/Linux + +``` +curl https://github.com/ahmet2mir/wildq/archive/1.0.6.zip -o wildq.zip +unzip wildq.zip -d ~~/bin~~ +rm -f wildq.zip +``` + +### MacOS + +``` +curl https://github.com/ahmet2mir/wildq/archive/1.0.6.zip -o wildq.zip +unzip wildq.zip -d ~/bin~ +rm -f wildq.zip +``` + +### Windows + +Wildq use [jq.py](https://github.com/mwilliamson/jq.py) and it's not yet available on windows platforms. +I tried to compile it without windows machine and I failed, and I don't had the time to try to understand how Windows / C binding / Python works. +If anybody would contribute, there is an [open issue](https://github.com/mwilliamson/jq.py/issues/20) (jq and onigurama are 'compilable' on Windows so I think that someone confortable with that OS could make it); + + +# Supported file types + +| type | color | ordering | output | source | +|------|-------|----------|--------|---------------------------------------------------------------------| +| hcl | json | no | json | [pyhcl](https://github.com/virtuald/pyhcl) by @virtuald | +| ini | yes | no | yes | [ConfigParser](https://docs.python.org/3/library/configparser.html) | +| json | yes | yes | yes | [json](https://docs.python.org/3/library/json.html) | +| toml | yes | no | yes | [toml](https://github.com/uiri/toml) by @uiri | +| xml | yes | no | yes | [xmldict](https://github.com/martinblech/xmltodict) by @martinblech | +| yaml | yes | yes | yes | [pyyaml](https://github.com/yaml/pyyaml) | + # Usage ``` -wildq [--yaml|--json|--toml|--ini|--xml|--hcl> [file] +$ wildq -i help +Usage: wildq [OPTIONS] [FILE] JQ_FILTER + +Options: + -c, --compact-output compact instead of pretty-printed output + -r, --raw output raw strings, not content texts + -C, --color-output colorize content (default), mutally + exclusive with --monochrome-output + + -M, --monochrome-output monochrome (don't colorize content), mutally + exclusive with --color-output + + --hcl Combine --input hcl --output json, mutally + exclusive with other Combined options + + --ini Combine --input ini --output json, mutally + exclusive with other Combined options + + --json Combine --input json --output json, mutally + exclusive with other Combined options + + --toml Combine --input toml --output json, mutally + exclusive with other Combined options + + --xml Combine --input xml --output json, mutally + exclusive with other Combined options + + --yaml Combine --input yaml --output json, mutally + exclusive with other Combined options + + -i, --input [hcl|ini|json|toml|xml|yaml] + Define the content type of file, mutally + exclusive with Combined option + + -o, --output [hcl|ini|json|toml|xml|yaml] + Define the content type of printed output, + mutally exclusive with Combined option + (default input format) + + --help Show this message and exit. ``` -There is also a shorter command `wq` comming with the package. +For backward compatibility in previous version only `--[yaml|json|toml|ini|xml|hcl]` was possible with default to json output. +We still keep Monochrome, raw and json output with thoses options. +Output was similar to `jq -MCr` (no color, no compact and no quote on single value) -Output is similar to `jq -MCr` (no color, no compact and no quote on single value) +But now, by default it's colorized, not raw and if you specify input using `-i` or `--input` output will be the same format. + +There is also a shorter command `wq` comming with the package. Like `jq cli`, wildq supports both of stdin and file to the function @@ -52,7 +133,7 @@ Content of `examples/json.json` ``` ```sh -cat examples/json.json | wildq --json ".keys[]" +cat examples/json.json | wildq -i json ".keys[]" { "key": "value1" } @@ -65,7 +146,7 @@ alone or ```sh -wildq --json ".keys[]" examples/json.json +wildq -i json ".keys[]" examples/json.json { "key": "value1" } @@ -77,7 +158,7 @@ alone or ```sh -wq --json ".keys[]" examples/json.json +wq -i json ".keys[]" examples/json.json { "key": "value1" } @@ -89,7 +170,7 @@ alone For TOML ```sh -cat examples/toml.toml | wildq --toml ".keys[]" +cat examples/toml.toml | wildq -i toml ".keys[]" { "key": "value1" } @@ -101,7 +182,7 @@ alone For INI (no array) ```sh -cat examples/ini.ini | wildq --ini ".keys" +cat examples/ini.ini | wildq -i ini ".keys" { "key1": "value1", "key2": "value2" @@ -110,7 +191,7 @@ cat examples/ini.ini | wildq --ini ".keys" For XML ```sh -cat examples/xml.xml | wildq --xml "." +cat examples/xml.xml | wildq -i xml "." { "root": { "general": { @@ -133,7 +214,7 @@ cat examples/xml.xml | wildq --xml "." For YAML ```sh -cat examples/yaml.yaml | wildq --yaml ".keys[]" +cat examples/yaml.yaml | wildq -i yaml ".keys[]" { "key1": "value1" } @@ -145,7 +226,7 @@ alone For HCL ```sh -cat examples/hcl.hcl | wildq --hcl ".keys[]" +cat examples/hcl.hcl | wildq -i hcl ".keys[]" { "key": "value1" } @@ -159,7 +240,7 @@ cat examples/hcl.hcl | wildq --hcl ".keys[]" Loop on keys in bash without creating a subshell ```sh -wildq --toml "keys[]" examples/toml.toml | while read -r key +wildq -i toml "keys[]" examples/toml.toml | while read -r key do echo "Getting key ${key}" done @@ -167,11 +248,13 @@ done ## TODO -- [ ] support all jq types -- [ ] add tests... -- [ ] add more control over filters and files +- [x] add tests... +- [x] add more control over filters and files +- [x] use click for the CLI +- [x] support different output - [ ] detect automagically filetype -- [ ] use click for the CLI +- [ ] support all jq types +- [ ] ordering ## Contributing diff --git a/docs/apireference.rst b/docs/apireference.rst new file mode 100644 index 0000000..d02f662 --- /dev/null +++ b/docs/apireference.rst @@ -0,0 +1,26 @@ +============= +API Reference +============= + +FileTypes +========= + +.. py:module:: wildq.filetypes + +.. automodule:: wildq.filetypes.hcl_type + :members: + +.. automodule:: wildq.filetypes.ini_type + :members: + +.. automodule:: wildq.filetypes.json_type + :members: + +.. automodule:: wildq.filetypes.toml_type + :members: + +.. automodule:: wildq.filetypes.xml_type + :members: + +.. automodule:: wildq.filetypes.yaml_type + :members: diff --git a/docs/changelog.rst b/docs/changelog.rst new file mode 100644 index 0000000..8e23b87 --- /dev/null +++ b/docs/changelog.rst @@ -0,0 +1,18 @@ +========= +Changelog +========= + +v1.1.0 - 12/10/2020 +=================== + +- use click for CLI +- add input and output format (close #GH-1) +- colorize output +- add tests and doc +- move from travis to github actions +- build using pyinstaller for linux and macos + + +v1.0.6 - 26/05/2020 +=================== +- First release diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..3e8c302 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# standard +import os +import sys +import datetime + +# third +import sphinx_rtd_theme + +sys.path.insert(0, os.path.abspath(".")) +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) + +# local +from wildq._wildq_version import version as release + +# -- Project information ----------------------------------------------------- + +year = datetime.datetime.now().year +project = "wildq - Command-line processor using jq c bindings" +copyright = "%d Ahmet Demir" % year +author = "Ahmet Demir (ahmet2mir)" + +# The short X.Y version +version = "" + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = ["sphinx.ext.autodoc", "sphinx.ext.todo", "sphinx.ext.napoleon"] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = ".rst" + +# The master toctree document. +master_doc = "index" + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "sphinx_rtd_theme" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = "wildqdoc" + + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, "wildq.tex", "wildq Documentation", "Ahmet Demir", "manual") +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [(master_doc, "wildq", "wildq Documentation", [author], 1)] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + master_doc, + "wildq", + "wildq Documentation", + author, + "wildq", + "One line description of project.", + "Miscellaneous", + ) +] + + +# -- Options for Epub output ------------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# +# epub_identifier = '' + +# A unique identification for the text. +# +# epub_uid = '' + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ["search.html"] + + +# -- Extension configuration ------------------------------------------------- + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + +# enable google docstring +napoleon_google_docstring = True +napoleon_numpy_docstring = False diff --git a/docs/getting-started.rst b/docs/getting-started.rst new file mode 100644 index 0000000..5958cb2 --- /dev/null +++ b/docs/getting-started.rst @@ -0,0 +1,14 @@ +Getting started +=============== + +For backward compatibility in previous version only `--[yaml|json|toml|ini|xml|hcl]` was possible with default to json output. +We still keep Monochrome, raw and json output with thoses options. +Output was similar to `jq -MCr` (no color, no compact and no quote on single value) + +But now, by default it's colorized, not raw and if you specify input using `-i` or `--input` output will be the same format. + +There is also a shorter command `wq` comming with the package. + +Like `jq cli`, wildq supports both of stdin and file to the function + +See examples to get some example. diff --git a/docs/guide/index.rst b/docs/guide/index.rst new file mode 100644 index 0000000..9b2ef7d --- /dev/null +++ b/docs/guide/index.rst @@ -0,0 +1,8 @@ +========== +User Guide +========== + +.. toctree:: + :maxdepth: 2 + + installing diff --git a/docs/guide/installing.rst b/docs/guide/installing.rst new file mode 100644 index 0000000..077a84c --- /dev/null +++ b/docs/guide/installing.rst @@ -0,0 +1,25 @@ +====================== +Installing wildq +====================== + +wildq is available on PyPI, so you can use :program:`pip`: + +.. code-block:: console + + $ pip install wildq + +Alternatively, if you don't have setuptools installed, `download it from PyPi +`_ and run + +.. code-block:: console + + $ python setup.py install + +To use the bleeding-edge version of wildq, you can get the source from +`GitHub `_ and install it as above: + +.. code-block:: console + + $ git clone git://github.com/ahmet2mir/wildq + $ cd wildq + $ python setup.py install diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..d9f7961 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,80 @@ +============================== +Wildq User Documentation +============================== + +Purpose of this package is to provide a simple wrapper arround jq for different formats. I'm tired of searching a package doing yaml jq, toml jq, ini jq etc. mainly used for scripting. + + +The latest stable version `is available on PyPI `_. + +.. code-block:: console + + pip install -U wildq + +:doc:`getting-started` + wildq's getting-started! + +:doc:`guide/index` + All detailed guide for wildq. + +:doc:`apireference` + The complete API documentation + +Community +--------- + +To get help with using wildq, create an issue on `GitHub Issues `_. + +Contributing +------------ + +**Yes please!** We are always looking for contributions, additions and improvements. + +The source is available on `GitHub `_ +and contributions are always encouraged. Contributions can be as simple as +minor tweaks to this documentation, the website or the core. + +To contribute, fork the project on `GitHub `_ +and send a pull request. + +Changes +------- + +See the :doc:`changelog` for a full list of changes to wildq. + +Offline Reading +--------------- + +Download the docs in `pdf `_ +or `epub `_ +formats for offline reading. + + +.. toctree:: + :maxdepth: 1 + :numbered: + :hidden: + + getting-started + guide/index + apireference + changelog + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + + +Licence +------- + +Copyright 2020 - Ahmet Demir + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +`http://www.apache.org/licenses/LICENSE-2.0 `_ + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/examples/toml.toml b/examples/toml.toml index a7718ef..abcfd10 100644 --- a/examples/toml.toml +++ b/examples/toml.toml @@ -1,5 +1,5 @@ -keys = [ "alone" ] +# keys = [ "alone" ] # produce an error on dump only [[keys]] key = "value1" diff --git a/examples/xml.xml b/examples/xml.xml index f928658..3962c43 100644 --- a/examples/xml.xml +++ b/examples/xml.xml @@ -12,4 +12,4 @@ alone - \ No newline at end of file + diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..78a7886 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,39 @@ +# Example configuration for Black. + +# NOTE: you have to use single-quoted strings in TOML for regular expressions. +# It's the equivalent of r-strings in Python. Multiline strings are treated as +# verbose regular expressions by Black. Use [ ] to denote a significant space +# character. + +[tool.black] +line-length = 80 +target-version = ['py36', 'py37', 'py38'] +include = '\.pyi?$' +exclude = ''' +/( + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.venv + | _build + | buck-out + | build + | dist + | tests/data +)/ +''' + +# Build system information below. +# NOTE: You don't need this in your own Black configuration. + +[build-system] +requires = ["setuptools>=41.0", "setuptools-scm", "wheel"] +build-backend = "setuptools.build_meta" + +# [tool.setuptools_scm] +# write_to = "wildq/_wildq_version.py" +# write_to_template = "version = \"{version}\"\n" +# version_scheme = "post-release" +# local_scheme = "no-local-version" diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..b29ffb8 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +; [options] +; setup_requires = setuptools_scm diff --git a/setup.py b/setup.py index cbc5f75..4cd81f6 100644 --- a/setup.py +++ b/setup.py @@ -17,20 +17,133 @@ """ __author__ = "Ahmet Demir " +import sys + +# exit immediatly +assert sys.version_info >= (3, 6, 0), "wildq requires Python 3.6+" + +import os +from pathlib import Path # noqa E402 from setuptools import setup, find_packages -from wildq.release import __version__ +from wildq._wildq_version import version + + +CURRENT_DIR = Path(__file__).parent +sys.path.insert(0, str(CURRENT_DIR)) # for setuptools.build_meta + setup( + extras_require={ + "dev": [ + "alabaster==0.7.12", + "altgraph==0.17", + "appdirs==1.4.4", + "attrs==20.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "babel==2.8.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "bandit==1.6.2", + "black==19.10b0", + "cached-property==1.5.2", + "cerberus==1.3.2", + "certifi==2020.6.20", + "cfgv==3.2.0; python_full_version >= '3.6.1'", + "chardet==3.0.4", + "click==7.1.2", + "colorama==0.4.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "coverage==5.3", + "distlib==0.3.1", + "docutils==0.16; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "filelock==3.0.12", + "flake8==3.8.4", + "flake8-bugbear==20.1.4", + "gitdb==4.0.5; python_version >= '3.4'", + "gitpython==3.1.9; python_version >= '3.4'", + "identify==1.5.6; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "idna==2.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "imagesize==1.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "importlib-metadata==2.0.0; python_version < '3.8'", + "importlib-resources==3.0.0; python_version < '3.7'", + "jinja2==2.11.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "markupsafe==1.1.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "mccabe==0.6.1", + "nodeenv==1.5.0", + "orderedmultidict==1.0.1", + "packaging==20.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "pathspec==0.8.0", + "pbr==5.5.0; python_version >= '2.6'", + "pep517==0.8.2", + "pip-shims==0.5.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "pipenv==2020.8.13", + "pipenv-setup==3.1.1", + "pipfile==0.0.2", + "plette[validation]==0.2.3; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "pre-commit==2.7.1", + "pycodestyle==2.6.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "pyflakes==2.2.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "pygments==2.7.1", + "pyinstaller==4.0", + "pyinstaller-hooks-contrib==2020.9", + "pyparsing==2.4.7; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "python-dateutil==2.8.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "pytz==2020.1", + "pyyaml==5.3.1", + "regex==2020.10.11", + "requests==2.24.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "requirementslib==1.5.13; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "six==1.15.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "smmap==3.0.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "snowballstemmer==2.0.0", + "sphinx==3.2.1", + "sphinx-rtd-theme==0.5.0", + "sphinxcontrib-applehelp==1.0.2; python_version >= '3.5'", + "sphinxcontrib-devhelp==1.0.2; python_version >= '3.5'", + "sphinxcontrib-htmlhelp==1.0.3; python_version >= '3.5'", + "sphinxcontrib-jsmath==1.0.1; python_version >= '3.5'", + "sphinxcontrib-qthelp==1.0.3; python_version >= '3.5'", + "sphinxcontrib-serializinghtml==1.1.4; python_version >= '3.5'", + "stevedore==3.2.2; python_version >= '3.6'", + "toml==0.10.1", + "tomlkit==0.7.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "typed-ast==1.4.1", + "typing==3.7.4.3; python_version < '3.7'", + "urllib3==1.25.10; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "virtualenv==20.0.34; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "virtualenv-clone==0.5.4; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "vistir==0.5.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "wheel==0.35.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "zipp==3.3.0; python_version < '3.8'", + ] + }, + install_requires=[ + "click==7.1.2", + "jq==1.1.1", + "pygments==2.7.1", + "pyhcl==0.4.4", + "pyyaml==5.3.1", + "toml==0.10.1", + "xmltodict==0.12.0", + ], name="wildq", - version=__version__, - long_description=open("README.md").read(), + # use_scm_version={"local_scheme": "no-local-version"}, + # setup_requires=["setuptools_scm"], + # py_modules=["_wildq_version"], + version=version, + description="Command-line TOML/JSON/INI/YAML/XML processor using jq c bindings.", + long_description=(CURRENT_DIR / "README.md").read_text(encoding="utf8"), long_description_content_type="text/markdown", - url="https://github.com/ahmet2mir/wildq.git", author="Ahmet Demir", author_email="me@ahmet2mir.eu", - description="Command-line TOML/JSON/INI/YAML/XML processor using jq c bindings", + url="https://github.com/ahmet2mir/wildq", + project_urls={ + "Changelog": "https://wildq.readthedocs.io/en/latest/changelog.html" + }, license="Apache 2.0", + + python_requires=">=3.6", + zip_safe=False, + entry_points={ + "console_scripts": ["wildq=wildq.wildq:main", "wq=wildq.__main__:main"] + }, keywords=[ "wildq", "jq", @@ -42,22 +155,13 @@ "parser", "shell", "hcl", + "color", + "highlight", ], - packages=find_packages(), - package_data={"": ["README.md"]}, - python_requires=">=3.5", - entry_points={ - "console_scripts": ["wildq=wildq.wildq:main", "wq=wildq.wildq:main"] - }, - install_requires=[ - "setuptools", - "PyYAML >= 3.11", - "toml >= 0.9.4", - "xmltodict >= 0.12.0", - "jq >= 1.0.1", - "pyhcl >= 0.4.4", - ], + test_suite="tests", classifiers=[ + "Development Status :: 4 - Beta", + "Environment :: Console", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent", @@ -65,6 +169,8 @@ "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3 :: Only", + "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Software Development :: Build Tools", ], ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_hcl.py b/tests/test_hcl.py new file mode 100644 index 0000000..62b8a88 --- /dev/null +++ b/tests/test_hcl.py @@ -0,0 +1,31 @@ +from unittest import TestCase, main + +from wildq.filetypes.hcl_type import loader, dumper, colorizer + + +class TestFiletypeHCL(TestCase): + def setUp(self): + self.data = """ +general { + user = "admin" +} + +keys { + key = "value1" +} + +keys { + key = "value2" +} +""" + + def test_loader(self): + assert loader(self.data)["general"]["user"] == "admin" + + def test_dumper(self): + dumped = dumper({"general": {"user": "admin"}}) + assert loader(dumped)["general"]["user"] == "admin" + + +if __name__ == "__main__": + main(module="test_hcl") diff --git a/tests/test_ini.py b/tests/test_ini.py new file mode 100644 index 0000000..34f389c --- /dev/null +++ b/tests/test_ini.py @@ -0,0 +1,26 @@ +from unittest import TestCase, main + +from wildq.filetypes.ini_type import loader, dumper, colorizer + + +class TestFiletypeINI(TestCase): + def setUp(self): + self.data = """ +[general] +user = admin + +[keys] +key1 = value1 +key2 = value2 +""" + + def test_loader(self): + assert loader(self.data)["general"]["user"] == "admin" + + def test_dumper(self): + dumped = dumper({"general": {"user": "admin"}}) + assert loader(dumped)["general"]["user"] == "admin" + + +if __name__ == "__main__": + main(module="test_ini") diff --git a/tests/test_json.py b/tests/test_json.py new file mode 100644 index 0000000..bebef52 --- /dev/null +++ b/tests/test_json.py @@ -0,0 +1,30 @@ +from unittest import TestCase, main + +from wildq.filetypes.json_type import loader, dumper, colorizer + + +class TestFiletypeJSON(TestCase): + def setUp(self): + self.data = """ +{ + "general": { + "user": "admin" + }, + "keys": [ + {"key": "value1"}, + {"key": "value2"}, + "alone" + ] +} +""" + + def test_loader(self): + assert loader(self.data)["general"]["user"] == "admin" + + def test_dumper(self): + dumped = dumper({"general": {"user": "admin"}}) + assert loader(dumped)["general"]["user"] == "admin" + + +if __name__ == "__main__": + main(module="test_json") diff --git a/tests/test_toml.py b/tests/test_toml.py new file mode 100644 index 0000000..27dbfbd --- /dev/null +++ b/tests/test_toml.py @@ -0,0 +1,28 @@ +from unittest import TestCase, main + +from wildq.filetypes.toml_type import loader, dumper, colorizer + + +class TestFiletypeTOML(TestCase): + def setUp(self): + self.data = """ + [[keys]] + key = "value2" + + [[keys]] + key = "value1" + + [general] + user = "admin" +""" + + def test_loader(self): + assert loader(self.data)["general"]["user"] == "admin" + + def test_dumper(self): + dumped = dumper({"general": {"user": "admin"}}) + assert loader(dumped)["general"]["user"] == "admin" + + +if __name__ == "__main__": + main(module="test_toml") diff --git a/tests/test_wild.py b/tests/test_wild.py index 8c6b1a7..e69de29 100644 --- a/tests/test_wild.py +++ b/tests/test_wild.py @@ -1,5 +0,0 @@ -from wildq import wildq - - -def test_usage(): - assert wildq.usage() == 0 diff --git a/tests/test_xml.py b/tests/test_xml.py new file mode 100644 index 0000000..546106b --- /dev/null +++ b/tests/test_xml.py @@ -0,0 +1,37 @@ +from unittest import TestCase, main + +from wildq.filetypes.xml_type import loader, dumper, colorizer + + +class TestFiletypeXML(TestCase): + def setUp(self): + self.data = """ + + + admin + + + + value1 + + + value2 + + alone + + +""" + + def test_loader(self): + assert loader(self.data)["root"]["general"]["user"] == "admin" + + def test_dumper(self): + dumped = dumper({"general": {"user": "admin"}}) + assert loader(dumped)["general"]["user"] == "admin" + + dumped = dumper({"general": {"user": "admin"}, "otherroot": 1}) + assert loader(dumped)["virtualroot"]["general"]["user"] == "admin" + + +if __name__ == "__main__": + main(module="test_xml") diff --git a/tests/test_yaml.py b/tests/test_yaml.py new file mode 100644 index 0000000..4b1509f --- /dev/null +++ b/tests/test_yaml.py @@ -0,0 +1,27 @@ +from unittest import TestCase, main + +from wildq.filetypes.yaml_type import loader, dumper, colorizer + + +class TestFiletypeYAML(TestCase): + def setUp(self): + self.data = """ +--- +keys: + - key1: value1 + - key2: value2 + - alone +general: + user: "admin" +""" + + def test_loader(self): + assert loader(self.data)["general"]["user"] == "admin" + + def test_dumper(self): + dumped = dumper({"general": {"user": "admin"}}) + assert loader(dumped)["general"]["user"] == "admin" + + +if __name__ == "__main__": + main(module="test_yaml") diff --git a/tests/tests.sh b/tests/tests.sh new file mode 100755 index 0000000..ab360a8 --- /dev/null +++ b/tests/tests.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +echo "Query with old format" +./dist/wildq --hcl examples/hcl.hcl . | jq . +./dist/wildq --ini examples/ini.ini . | jq . +./dist/wildq --json examples/json.json . | jq . +./dist/wildq --toml examples/toml.toml . | jq . +./dist/wildq --xml examples/xml.xml . | jq . +./dist/wildq --yaml examples/yaml.yaml . | jq . + +echo "Query with same in and out" +./dist/wildq -i hcl examples/hcl.hcl . +./dist/wildq -i ini examples/ini.ini . +./dist/wildq -i json examples/json.json . +./dist/wildq -i toml examples/toml.toml . +./dist/wildq -i xml examples/xml.xml . +./dist/wildq -i yaml examples/yaml.yaml . + +echo "Query with json out" +./dist/wildq -i hcl -o json examples/hcl.hcl . +./dist/wildq -i ini -o json examples/ini.ini . +./dist/wildq -i json -o json examples/json.json . +./dist/wildq -i toml -o json examples/toml.toml . +./dist/wildq -i xml -o json examples/xml.xml . +./dist/wildq -i yaml -o json examples/yaml.yaml . + +echo "Query with yaml out" +./dist/wildq -i hcl -o yaml examples/hcl.hcl . +./dist/wildq -i ini -o yaml examples/ini.ini . +./dist/wildq -i json -o yaml examples/json.json . +./dist/wildq -i toml -o yaml examples/toml.toml . +./dist/wildq -i xml -o yaml examples/xml.xml . +./dist/wildq -i yaml -o yaml examples/yaml.yaml . diff --git a/tox.ini b/tox.ini deleted file mode 100644 index bb62161..0000000 --- a/tox.ini +++ /dev/null @@ -1,45 +0,0 @@ -# Tox (http://tox.testrun.org/) is a tool for running tests -# in multiple virtualenvs. This configuration file will run the -# test suite on all supported python versions. To use it, "pip install tox" -# and then run "tox" from this directory. - -[tox] -skipsdist = True -skip_missing_interpreters = True -envlist = py36,py37,py38 - -[testenv] -setenv = - PYTHONPATH = {toxinidir} -deps = - sphinx-rtd-theme - pytest - mock - pytest-cov - pylint - sphinx - PyYAML >= 3.11 - toml >= 0.9.4 - xmltodict >= 0.12.0 - jq >= 1.0.1 - pyhcl >= 0.4.4 -commands = - py.test -vv --capture=fd --ignore='.tox' --cov-report term-missing --cov wildq - -[testenv:lint] -deps = - pylint -commands = - - pylint ./wildq - -[testenv:deploy] -commands = - python setup.py install -n -v - python setup.py register -r pypi - python setup.py bdist_wheel upload -r pypi - -[testenv:format] -deps = - black -commands = - black wildq/*.py diff --git a/wildq/__main__.py b/wildq/__main__.py index bcd75f7..89c07cf 100644 --- a/wildq/__main__.py +++ b/wildq/__main__.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # coding: utf-8 -from wildq.wildq import main +from wildq.cli import cli if __name__ == "__main__": - main() + cli() diff --git a/wildq/_wildq_version.py b/wildq/_wildq_version.py new file mode 100644 index 0000000..b2b60a5 --- /dev/null +++ b/wildq/_wildq_version.py @@ -0,0 +1 @@ +version = "1.1.0" diff --git a/wildq/cli.py b/wildq/cli.py new file mode 100644 index 0000000..cca0663 --- /dev/null +++ b/wildq/cli.py @@ -0,0 +1,197 @@ +# standard +import sys + +# third +import jq +import click + +# local +import wildq.filetypes + +# could be replaced by importlib / getattr but explicit is better than implicit +SUPPORTED_FILETYPES = { + "hcl": wildq.filetypes.hcl_type, + "ini": wildq.filetypes.ini_type, + "json": wildq.filetypes.json_type, + "toml": wildq.filetypes.toml_type, + "xml": wildq.filetypes.xml_type, + "yaml": wildq.filetypes.yaml_type, +} + + +def compiler(jq_filter, data): + return jq.compile(jq_filter).input(data) + + +@click.command() +# @click.option('--input', help='compose file to work with', type=click.File('r'), default=sys.stdin) +# @click.option('--output', help='compose file to work with', type=click.File('r'), default=sys.stdin) +@click.option( + "-c", + "--compact-output", + is_flag=True, + default=False, + help="compact instead of pretty-printed output", +) +@click.option( + "-r", + "--raw", + is_flag=True, + default=False, + help="output raw strings, not content texts", +) +@click.option( + "-C", + "--color-output", + is_flag=True, + default=False, + help="colorize content (default), mutally exclusive with --monochrome-output", +) +@click.option( + "-M", + "--monochrome-output", + is_flag=True, + default=False, + help="monochrome (don't colorize content), mutally exclusive with --color-output", +) +# @click.option('-S', '--sort', is_flag=True, default=False, help="sort keys of objects on output") +@click.option( + "--hcl", + is_flag=True, + default=False, + help="Combine --input hcl --output hcl, mutally exclusive with other Combined options", +) +@click.option( + "--ini", + is_flag=True, + default=False, + help="Combine --input ini --output ini, mutally exclusive with other Combined options", +) +@click.option( + "--json", + is_flag=True, + default=False, + help="Combine --input json --output json, mutally exclusive with other Combined options", +) +@click.option( + "--toml", + is_flag=True, + default=False, + help="Combine --input toml --output toml, mutally exclusive with other Combined options", +) +@click.option( + "--xml", + is_flag=True, + default=False, + help="Combine --input xml --output xml, mutally exclusive with other Combined options", +) +@click.option( + "--yaml", + is_flag=True, + default=False, + help="Combine --input yaml --output yaml, mutally exclusive with other Combined options", +) +@click.option( + "-i", + "--input", + type=click.Choice(list(SUPPORTED_FILETYPES.keys())), + help="Define the content type of file, mutally exclusive with Combined option", +) +@click.option( + "-o", + "--output", + type=click.Choice(list(SUPPORTED_FILETYPES.keys())), + default=None, + show_default=False, + help="Define the content type of printed output, mutally exclusive with Combined option (default input format)", +) +@click.argument("file", type=click.File("r"), default=sys.stdin) +@click.argument("jq_filter") +def cli(*args, **kwargs): + from pprint import pprint + + if (kwargs["input"] or kwargs["output"]) and any( + [True for x in list(SUPPORTED_FILETYPES.keys()) if kwargs[x]] + ): + print( + "When using combined option you can't use --input or --output, please use them explictly" + ) + sys.exit(1) + + if kwargs["color_output"] and kwargs["monochrome_output"]: + print( + "--color-output and --monochrome-output are mutally exclusive, select one" + ) + sys.exit(1) + + colorized = False + if not kwargs["color_output"] and not kwargs["monochrome_output"]: + colorized = True + + if kwargs["color_output"]: + colorized = True + + if kwargs["monochrome_output"]: + colorized = False + + input_fmt = None + output_fmt = None + + raw = kwargs["raw"] + + # backward compatibility, if a type is specified without -i, output to json + # enforce colorize to false and raw to true like first release of wq + for spt in list(SUPPORTED_FILETYPES.keys()): + if kwargs[spt]: + input_fmt = spt + output_fmt = "json" + colorized = False + raw = True + + # user specified input otherwise raise + if not input_fmt: + input_fmt = kwargs["input"] + + # if user specify input but not output, use default input format + if not output_fmt: + if kwargs.get("output") is None: + output_fmt = input_fmt + else: + output_fmt = kwargs["output"] + + if not input_fmt or not output_fmt: + if input_fmt is None: + print("input option is empty") + if output_fmt is None: + print("output option is empty") + print( + "Issue when resolving input and output, getting confiuration %s", + str(kwargs), + ) + sys.exit(1) + + module_in = SUPPORTED_FILETYPES[input_fmt] + module_out = SUPPORTED_FILETYPES[output_fmt] + + content = module_in.loader(kwargs["file"].read()) + + # skip silently if content is empty + if not content: + return 0 + + compiled = compiler(kwargs["jq_filter"], content) + + for line in compiled: + if not isinstance(line, dict) and not isinstance(line, list): + if raw: + print(line.strip('"')) + else: + print(line) + continue + + dumped = module_out.dumper(line) + + if colorized: + print(module_out.colorizer(dumped)) + else: + print(dumped) diff --git a/wildq/filetypes/__init__.py b/wildq/filetypes/__init__.py new file mode 100644 index 0000000..246d12c --- /dev/null +++ b/wildq/filetypes/__init__.py @@ -0,0 +1,54 @@ +# coding: utf-8 +from wildq.filetypes import hcl_type +from wildq.filetypes import ini_type +from wildq.filetypes import json_type +from wildq.filetypes import toml_type +from wildq.filetypes import xml_type +from wildq.filetypes import yaml_type + +# Auto Doc +for ft in [hcl_type, ini_type, json_type, toml_type, xml_type, yaml_type]: + ft.loader.__doc__ = f"""Pass data as string to {ft.FILETYPE_NAME} loader. + + Args: + data_string (str): String content to load, usually a file content. + + Returns: + Loaded content. Return type depends on content format, + should be dict, list, string etc. + + Examples: + >>> from wildq.filetypes import {ft.FILETYPE_NAME.lower()}_type + >>> with open("examples/{ft.FILETYPE_NAME.lower()}.{ft.FILETYPE_NAME.lower()}", "r") as fd: + ... data_string = fd.read() + >>> data = {ft.FILETYPE_NAME.lower()}_type.loader(data_string) + {ft.FILETYPE_EXAMPLE_LOADER} + """ + + ft.dumper.__doc__ = f"""Pass loaded data to {ft.FILETYPE_NAME} dumper. + + Args: + data: Should be dict, list, string etc. + + Returns: + str: Dumped data. + + Examples: + >>> from wildq.filetypes import {ft.FILETYPE_NAME.lower()}_type + >>> data = {ft.FILETYPE_NAME.lower()}_type.dumper(data_string) + {ft.FILETYPE_EXAMPLE_DUMPER} + """ + + ft.colorizer.__doc__ = f"""Pass dumped data to {ft.FILETYPE_NAME} colorize. + + Args: + content (str): string representation of data, usually a dumped data. + + Returns: + str: Colorized data. + + Examples: + >>> from wildq.filetypes import {ft.FILETYPE_NAME.lower()}_type + >>> data = {ft.FILETYPE_NAME.lower()}_type.colorizer(data_string) + ... + """ diff --git a/wildq/filetypes/hcl_type.py b/wildq/filetypes/hcl_type.py new file mode 100644 index 0000000..d5addc7 --- /dev/null +++ b/wildq/filetypes/hcl_type.py @@ -0,0 +1,49 @@ +# coding: utf-8 +"""HCL type""" + +# standard +import json +from collections import OrderedDict + +# third +import hcl +from pygments import highlight, lexers, formatters + +FILETYPE_NAME = "HCL" +FILETYPE_EXAMPLE_LOADER = """{ + 'general': { + 'user': 'admin' + }, + 'keys': [ + {'key': 'value1'}, + {'key': 'value2'} + ] + } +""" +FILETYPE_EXAMPLE_DUMPER = """general { + user = "admin" + } + keys { + key = "value1" + } + keys { + key = "value2" + } +""" + + +def loader(data_string): + return hcl.loads(data_string) + + +def dumper(data): + return json.dumps(data, indent=4) + + +def colorizer(content): + # dump to hcl is not supported, so fallback to json + return highlight( + content, + lexers.JsonLexer(), # lexers.TerraformLexer() + formatters.TerminalFormatter(), + ) diff --git a/wildq/filetypes/ini_type.py b/wildq/filetypes/ini_type.py new file mode 100644 index 0000000..64b1398 --- /dev/null +++ b/wildq/filetypes/ini_type.py @@ -0,0 +1,49 @@ +# coding: utf-8 +"""INI type""" + +# standard +import io +import sys +import configparser +from collections import OrderedDict + +# third +from pygments import highlight, lexers, formatters + +FILETYPE_NAME = "INI" +FILETYPE_EXAMPLE_LOADER = """OrderedDict([ + ('general', OrderedDict([ + ('user', 'admin') + ])), + ('keys', OrderedDict([ + ('key1', 'value1'), + ('key2', 'value2') + ])) + ]) +""" +FILETYPE_EXAMPLE_DUMPER = """[general] + user = admin + [keys] + key1 = value1 + key2 = value2 +""" + + +def loader(data_string): + config = configparser.ConfigParser() + config.read_string(data_string) + return config._sections + + +def dumper(data): + parser = configparser.ConfigParser() + parser.read_dict(data) + + buf = io.StringIO() + parser.write(buf) + + return buf.getvalue() + + +def colorizer(content): + return highlight(content, lexers.IniLexer(), formatters.TerminalFormatter()) diff --git a/wildq/filetypes/json_type.py b/wildq/filetypes/json_type.py new file mode 100644 index 0000000..356476b --- /dev/null +++ b/wildq/filetypes/json_type.py @@ -0,0 +1,47 @@ +# coding: utf-8 +"""JSON type""" + +# standard +import json +from collections import OrderedDict + +# third +from pygments import highlight, lexers, formatters + +FILETYPE_NAME = "JSON" +FILETYPE_EXAMPLE_LOADER = """{ + 'general': { + 'user': 'admin' + }, + 'keys': [ + {'key': 'value1'}, + {'key': 'value2'}, + 'alone' + ] + } +""" +FILETYPE_EXAMPLE_DUMPER = """{ + "general": { + "user": "admin" + }, + "keys": [ + {"key": "value1"}, + {"key": "value2"}, + "alone" + ] + } +""" + + +def loader(data_string): + return json.loads(data_string) + + +def dumper(data): + return json.dumps(data, indent=4) + + +def colorizer(content): + return highlight( + content, lexers.JsonLexer(), formatters.TerminalFormatter() + ) diff --git a/wildq/filetypes/toml_type.py b/wildq/filetypes/toml_type.py new file mode 100644 index 0000000..29d86d2 --- /dev/null +++ b/wildq/filetypes/toml_type.py @@ -0,0 +1,43 @@ +# coding: utf-8 +"""TOML type""" + +# standard +from collections import OrderedDict + +# third +import toml +from pygments import highlight, lexers, formatters + +FILETYPE_NAME = "TOML" +FILETYPE_EXAMPLE_LOADER = """{ + 'general': { + 'user': 'admin' + }, + 'keys': [ + {'key': 'value1'}, + {'key': 'value2'} + ] + } +""" +FILETYPE_EXAMPLE_DUMPER = """# keys = [ "alone" ] # produce an error on dump only + [[keys]] + key = "value1" + [[keys]] + key = "value2" + [general] + user = "admin" +""" + + +def loader(data_string): + return toml.loads(data_string) + + +def dumper(data): + return toml.dumps(data) + + +def colorizer(content): + return highlight( + content, lexers.TOMLLexer(), formatters.TerminalFormatter() + ) diff --git a/wildq/filetypes/xml_type.py b/wildq/filetypes/xml_type.py new file mode 100644 index 0000000..85aa3b7 --- /dev/null +++ b/wildq/filetypes/xml_type.py @@ -0,0 +1,61 @@ +# coding: utf-8 +"""XML type""" + +import json +from collections import OrderedDict + +import xmltodict +from pygments import highlight, lexers, formatters + +FILETYPE_NAME = "XML" +FILETYPE_EXAMPLE_LOADER = """OrderedDict([ + ('root', OrderedDict([ + ('general', OrderedDict([ + ('user', 'admin') + ])), + ('keys', OrderedDict([ + ('element', [ + OrderedDict([('key', 'value1')]), + OrderedDict([('key', 'value2')]), + 'alone' + ]) + ])) + ])) + ]) +""" +FILETYPE_EXAMPLE_DUMPER = """ + + + admin + + + + value1 + + + value2 + + alone + + +""" + + +# local + + +def loader(data_string): + return xmltodict.parse(data_string) + + +def dumper(data): + if isinstance(data, dict) and len(data.keys()) == 1: + return xmltodict.unparse(data, pretty=True, indent=" ") + else: + return xmltodict.unparse( + {"virtualroot": data}, pretty=True, indent=" " + ) + + +def colorizer(content): + return highlight(content, lexers.XmlLexer(), formatters.TerminalFormatter()) diff --git a/wildq/filetypes/yaml_type.py b/wildq/filetypes/yaml_type.py new file mode 100644 index 0000000..3c1ef48 --- /dev/null +++ b/wildq/filetypes/yaml_type.py @@ -0,0 +1,74 @@ +# coding: utf-8 +"""YAML type""" + +# standard +from collections import OrderedDict + +# third +import yaml +from pygments import highlight, lexers, formatters + +FILETYPE_NAME = "YAML" +FILETYPE_EXAMPLE_LOADER = """{ + 'general': { + 'user': 'admin' + }, + 'keys': [ + {'key': 'value1'}, + {'key': 'value2'}, + 'alone' + ] + } +""" +FILETYPE_EXAMPLE_DUMPER = """general: + user: "admin" + keys: + - key1: value1 + - key2: value2 + - alone +""" + + +def loader(data_string): + return yaml.safe_load(data_string) + + +def dumper(data): + # Dump YAML thanks to https://github.com/dale3h/python-lovelace. + def ordered_dump(content, stream=None, Dumper=yaml.Dumper, **kwargs): + """YAML dumper for OrderedDict.""" + + class OrderedDumper(Dumper): + """Wrapper class for YAML dumper.""" + + def ignore_aliases(self, content): + """Disable aliases in YAML dump.""" + return True + + def increase_indent(self, flow=False, indentless=False): + """Increase indent on YAML lists.""" + return super(OrderedDumper, self).increase_indent(flow, False) + + def _dict_representer(dumper, content): + """Function to represent OrderDict and derivitives.""" + return dumper.represent_mapping( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, content.items() + ) + + OrderedDumper.add_multi_representer(OrderedDict, _dict_representer) + + OrderedDumper.add_representer( + str, yaml.representer.SafeRepresenter.represent_str + ) + + return yaml.dump(content, stream, OrderedDumper, **kwargs) + + return ordered_dump( + content=data, Dumper=yaml.SafeDumper, default_flow_style=False + ) + + +def colorizer(content): + return highlight( + content, lexers.YamlLexer(), formatters.TerminalFormatter() + ) diff --git a/wildq/release.py b/wildq/release.py deleted file mode 100644 index 382021f..0000000 --- a/wildq/release.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "1.0.6" diff --git a/wildq/wildq.py b/wildq/wildq.py deleted file mode 100644 index dcbc304..0000000 --- a/wildq/wildq.py +++ /dev/null @@ -1,79 +0,0 @@ -# coding: utf-8 -import io -import os -import sys -import json -import configparser - -# third parties -import jq -import hcl -import yaml -import toml -import xmltodict - - -def usage(): - print("wildq <--yaml|--json|--toml|--ini|--xml|--hcl> [file]") - return 0 - - -def ini_load_string(string): - config = configparser.ConfigParser() - config.read_string(string) - return config._sections - - -def run(filetype, jq_filter, content): - - filetypes = { - "--yaml": yaml.safe_load, - "--json": json.loads, - "--toml": toml.loads, - "--ini": ini_load_string, - "--xml": xmltodict.parse, - "--hcl": hcl.loads, - } - - data = filetypes[filetype](content) - - for item in jq.compile(jq_filter).input(data): - print(json.dumps(item, indent=4, sort_keys=True).strip('"')) - - -def cli(args): - if len(args) < 3: - usage() - return 1 - - if args[1] not in ["--yaml", "--json", "--toml", "--ini", "--xml", "--hcl"]: - print("Bad format " + args[1]) - usage() - return 2 - - file = sys.stdin - if len(sys.argv) >= 4: - if not os.path.exists(sys.argv[3]): - print("Unable to open file " + sys.argv[3]) - return 1 - file = sys.argv[3] - - if isinstance(file, io.TextIOWrapper): - content = file.read() - else: - with open(file, "r") as fd: - content = fd.read() - - # skip silently if content is empty - if not content: - return 0 - - return run(sys.argv[1], sys.argv[2], content) - - -def main(): - sys.exit(cli(sys.argv)) - - -if __name__ == "__main__": - main()