From 8ad19cc6a5370a0fd2e1ca1bb1e1f248e0caa03b Mon Sep 17 00:00:00 2001 From: hdoupe Date: Wed, 18 Sep 2019 12:31:42 -0400 Subject: [PATCH 01/18] Add module for taking screenshots of renderable outputs --- cs_storage/__init__.py | 6 +- cs_storage/screenshot.py | 105 +++++ cs_storage/templates/index.html | 39 ++ cs_storage/tests/test-tb-remote.json | 595 +++++++++++++++++++++++++++ cs_storage/tests/test-tc-remote.json | 50 +++ cs_storage/tests/test_screenshot.py | 29 ++ environment.yml | 3 + 7 files changed, 826 insertions(+), 1 deletion(-) create mode 100644 cs_storage/screenshot.py create mode 100644 cs_storage/templates/index.html create mode 100644 cs_storage/tests/test-tb-remote.json create mode 100644 cs_storage/tests/test-tc-remote.json create mode 100644 cs_storage/tests/test_screenshot.py diff --git a/cs_storage/__init__.py b/cs_storage/__init__.py index 6684515..136f91f 100644 --- a/cs_storage/__init__.py +++ b/cs_storage/__init__.py @@ -10,6 +10,8 @@ from marshmallow import Schema, fields, validate +from .screenshot import screenshot + __version__ = "1.6.0" @@ -61,7 +63,7 @@ def get_serializer(media_type): class Output: """Output mixin shared among LocalOutput and RemoteOutput""" - + id = fields.UUID() title = fields.Str() media_type = fields.Str( validate=validate.OneOf(choices=["bokeh", "table", "CSV", "PNG", "JPEG", "MP3", "MP4", "HDF5", "PDF", "Markdown", "Text"]) @@ -115,6 +117,7 @@ def write(task_id, loc_result, do_upload=True): zipfileobj.writestr(filename, ser) rem_result[category]["outputs"].append( { + # "id": uuid.uuid4(), "title": output["title"], "media_type": output["media_type"], "filename": filename, @@ -148,6 +151,7 @@ def read(rem_result): rem_data = ser.deserialize(zipfileobj.read(rem_output["filename"])) read[category].append( { + # "id": rem_output["unique_id"], "title": rem_output["title"], "media_type": rem_output["media_type"], "data": rem_data, diff --git a/cs_storage/screenshot.py b/cs_storage/screenshot.py new file mode 100644 index 0000000..1c73d38 --- /dev/null +++ b/cs_storage/screenshot.py @@ -0,0 +1,105 @@ +import asyncio +import os +import tempfile + +try: + # These dependencies are optional. The storage component may be used + # without the screenshot component. + from jinja2 import Template + from pyppeteer import launch + from bokeh.resources import CDN + BASE_ARGS = {"bokeh_scripts": {"cdn_js": CDN.js_files[0], "widget_js": CDN.js_files[1]}} + +except ImportError: + Template = None + launch = None + CDN = None + BASE_ARGS = {} + +import cs_storage + + + + +CURRENT_DIR = os.path.abspath(os.path.dirname(__file__)) + + +class ScreenshotError(Exception): + pass + + +def get_template(): + if Template is None: + return None + with open(f"{CURRENT_DIR}/templates/index.html", "r") as f: + text = f.read() + + template = Template(text) + + return template + + +TEMPLATE = get_template() + + +def write_template(output): + kwargs = {**BASE_ARGS, **{"output": output}} + return TEMPLATE.render(**kwargs) + + +async def _screenshot(template_path, pic_path): + """ + Use pyppeteer, a python port of puppeteer, to open the + template at template_path and take a screenshot of the + output that is rendered within it. + + The output is rendered within a Bootstrap card element. + This element is only as big as the elements that it contains. + Thus, we only need to get the dimensions of the bootstrap + card to figure out which part of the screen we need to use + for the screenshot! + + Note: pyppetter looks stale. If it continues to not be + maintained well, then the extremely active, well-maintained + puppeteer should be used for creating these screenshots. The + downside of using puppeteer is that it is written in nodejs. + """ + browser = await launch() + page = await browser.newPage() + await page.goto(f"file://{template_path}") + await page.setViewport(dict(width=1920, height=1080)) + await page.waitFor(1000) + element = await page.querySelector("#output") + if element is None: + raise ScreenshotError("Unable to take screenshot.") + boundingbox = await element.boundingBox() + clip = dict( + x=boundingbox["x"], + y=boundingbox["y"], + width=min(boundingbox["width"], 1920), + height=min(boundingbox["height"], 1080), + ) + await page.screenshot(path=f"{pic_path}", type="png", clip=clip) + await browser.close() + + +def screenshot(output): + """ + Create screenshot of outputs. The intermediate results are + written to temporary files and a picture, represented as a + stream of bytes, is returned. + """ + if launch is None: + return None + html = write_template(output) + with tempfile.NamedTemporaryFile(suffix=".html") as temp: + temp.write(html.encode("utf-8")) + temp.seek(0) + template_path = temp.name + with tempfile.NamedTemporaryFile(suffix=".png") as pic: + pic_path = pic.name + asyncio.get_event_loop().run_until_complete( + _screenshot(template_path, pic_path) + ) + pic_bytes = pic.read() + return pic_bytes diff --git a/cs_storage/templates/index.html b/cs_storage/templates/index.html new file mode 100644 index 0000000..d25327e --- /dev/null +++ b/cs_storage/templates/index.html @@ -0,0 +1,39 @@ + + + + Compute Studio + + + + + + + + + + + + + + + + +
+
+
+

{{output.title}}

+ {% if output.media_type == 'bokeh' %} +
+ {{output.data.html|safe}} + {{output.data.javascript|safe}} +
+ {% elif output.media_type == 'table' %} +
+ {{output.data|safe}} +
+ {% endif %} +
+
+
+ + diff --git a/cs_storage/tests/test-tb-remote.json b/cs_storage/tests/test-tb-remote.json new file mode 100644 index 0000000..142d8bf --- /dev/null +++ b/cs_storage/tests/test-tb-remote.json @@ -0,0 +1,595 @@ +{ + "outputs": { + "renderable": { + "outputs": [ + { + "title": "", + "filename": ".json", + "media_type": "bokeh" + }, + { + "title": "Aggregate Results", + "filename": "Aggregate Results.json", + "media_type": "bokeh" + }, + { + "title": "Tables", + "filename": "Tables.json", + "media_type": "bokeh" + } + ], + "ziplocation": "16fd7144-08f7-4b77-b239-8012fa1fa8b5_renderable.zip" + }, + "downloadable": { + "outputs": [ + { + "title": "Total Liabilities Change by Calendar Year (Billions).csv", + "filename": "Total Liabilities Change by Calendar Year (Billions).csv", + "media_type": "CSV" + }, + { + "title": "Total Liabilities Baseline by Calendar Year (Billions).csv", + "filename": "Total Liabilities Baseline by Calendar Year (Billions).csv", + "media_type": "CSV" + }, + { + "title": "Total Liabilities Reform by Calendar Year (Billions).csv", + "filename": "Total Liabilities Reform by Calendar Year (Billions).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income bin (2019).csv", + "filename": "Base plan tax vars, weighted total by expanded income bin (2019).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income bin (2020).csv", + "filename": "Base plan tax vars, weighted total by expanded income bin (2020).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income bin (2021).csv", + "filename": "Base plan tax vars, weighted total by expanded income bin (2021).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income bin (2022).csv", + "filename": "Base plan tax vars, weighted total by expanded income bin (2022).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income bin (2023).csv", + "filename": "Base plan tax vars, weighted total by expanded income bin (2023).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income bin (2024).csv", + "filename": "Base plan tax vars, weighted total by expanded income bin (2024).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income bin (2025).csv", + "filename": "Base plan tax vars, weighted total by expanded income bin (2025).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income bin (2026).csv", + "filename": "Base plan tax vars, weighted total by expanded income bin (2026).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income bin (2027).csv", + "filename": "Base plan tax vars, weighted total by expanded income bin (2027).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income bin (2028).csv", + "filename": "Base plan tax vars, weighted total by expanded income bin (2028).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income bin (2029).csv", + "filename": "Base plan tax vars, weighted total by expanded income bin (2029).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income bin (2019).csv", + "filename": "User plan tax vars, weighted total by expanded income bin (2019).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income bin (2020).csv", + "filename": "User plan tax vars, weighted total by expanded income bin (2020).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income bin (2021).csv", + "filename": "User plan tax vars, weighted total by expanded income bin (2021).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income bin (2022).csv", + "filename": "User plan tax vars, weighted total by expanded income bin (2022).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income bin (2023).csv", + "filename": "User plan tax vars, weighted total by expanded income bin (2023).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income bin (2024).csv", + "filename": "User plan tax vars, weighted total by expanded income bin (2024).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income bin (2025).csv", + "filename": "User plan tax vars, weighted total by expanded income bin (2025).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income bin (2026).csv", + "filename": "User plan tax vars, weighted total by expanded income bin (2026).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income bin (2027).csv", + "filename": "User plan tax vars, weighted total by expanded income bin (2027).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income bin (2028).csv", + "filename": "User plan tax vars, weighted total by expanded income bin (2028).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income bin (2029).csv", + "filename": "User plan tax vars, weighted total by expanded income bin (2029).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2019).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2019).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2020).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2020).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2021).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2021).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2022).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2022).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2023).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2023).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2024).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2024).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2025).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2025).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2026).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2026).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2027).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2027).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2028).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2028).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2029).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2029).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2019).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2019).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2020).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2020).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2021).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2021).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2022).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2022).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2023).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2023).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2024).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2024).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2025).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2025).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2026).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2026).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2027).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2027).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2028).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2028).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2029).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2029).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2019).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2019).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2020).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2020).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2021).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2021).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2022).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2022).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2023).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2023).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2024).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2024).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2025).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2025).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2026).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2026).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2027).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2027).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2028).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2028).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2029).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2029).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income decile (2019).csv", + "filename": "Base plan tax vars, weighted total by expanded income decile (2019).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income decile (2020).csv", + "filename": "Base plan tax vars, weighted total by expanded income decile (2020).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income decile (2021).csv", + "filename": "Base plan tax vars, weighted total by expanded income decile (2021).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income decile (2022).csv", + "filename": "Base plan tax vars, weighted total by expanded income decile (2022).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income decile (2023).csv", + "filename": "Base plan tax vars, weighted total by expanded income decile (2023).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income decile (2024).csv", + "filename": "Base plan tax vars, weighted total by expanded income decile (2024).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income decile (2025).csv", + "filename": "Base plan tax vars, weighted total by expanded income decile (2025).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income decile (2026).csv", + "filename": "Base plan tax vars, weighted total by expanded income decile (2026).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income decile (2027).csv", + "filename": "Base plan tax vars, weighted total by expanded income decile (2027).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income decile (2028).csv", + "filename": "Base plan tax vars, weighted total by expanded income decile (2028).csv", + "media_type": "CSV" + }, + { + "title": "Base plan tax vars, weighted total by expanded income decile (2029).csv", + "filename": "Base plan tax vars, weighted total by expanded income decile (2029).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income decile (2019).csv", + "filename": "User plan tax vars, weighted total by expanded income decile (2019).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income decile (2020).csv", + "filename": "User plan tax vars, weighted total by expanded income decile (2020).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income decile (2021).csv", + "filename": "User plan tax vars, weighted total by expanded income decile (2021).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income decile (2022).csv", + "filename": "User plan tax vars, weighted total by expanded income decile (2022).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income decile (2023).csv", + "filename": "User plan tax vars, weighted total by expanded income decile (2023).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income decile (2024).csv", + "filename": "User plan tax vars, weighted total by expanded income decile (2024).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income decile (2025).csv", + "filename": "User plan tax vars, weighted total by expanded income decile (2025).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income decile (2026).csv", + "filename": "User plan tax vars, weighted total by expanded income decile (2026).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income decile (2027).csv", + "filename": "User plan tax vars, weighted total by expanded income decile (2027).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income decile (2028).csv", + "filename": "User plan tax vars, weighted total by expanded income decile (2028).csv", + "media_type": "CSV" + }, + { + "title": "User plan tax vars, weighted total by expanded income decile (2029).csv", + "filename": "User plan tax vars, weighted total by expanded income decile (2029).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2019).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2019).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2020).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2020).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2021).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2021).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2022).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2022).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2023).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2023).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2024).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2024).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2025).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2025).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2026).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2026).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2027).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2027).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2028).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2028).csv", + "media_type": "CSV" + }, + { + "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2029).csv", + "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2029).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2019).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2019).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2020).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2020).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2021).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2021).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2022).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2022).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2023).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2023).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2024).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2024).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2025).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2025).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2026).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2026).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2027).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2027).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2028).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2028).csv", + "media_type": "CSV" + }, + { + "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2029).csv", + "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2029).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2019).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2019).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2020).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2020).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2021).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2021).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2022).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2022).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2023).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2023).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2024).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2024).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2025).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2025).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2026).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2026).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2027).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2027).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2028).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2028).csv", + "media_type": "CSV" + }, + { + "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2029).csv", + "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2029).csv", + "media_type": "CSV" + } + ], + "ziplocation": "16fd7144-08f7-4b77-b239-8012fa1fa8b5_downloadable.zip" + } + }, + "version": "v1" +} \ No newline at end of file diff --git a/cs_storage/tests/test-tc-remote.json b/cs_storage/tests/test-tc-remote.json new file mode 100644 index 0000000..2e648a2 --- /dev/null +++ b/cs_storage/tests/test-tc-remote.json @@ -0,0 +1,50 @@ +{ + "outputs": { + "renderable": { + "outputs": [ + { + "title": "Basic Liabilities", + "filename": "Basic Liabilities.html", + "media_type": "table" + }, + { + "title": "Tax Liabilities by Wage (Holding Other Inputs Constant)", + "filename": "Tax Liabilities by Wage (Holding Other Inputs Constant).json", + "media_type": "bokeh" + }, + { + "title": "Tax Rates by Wage (Holding Other Inputs Constant)", + "filename": "Tax Rates by Wage (Holding Other Inputs Constant).json", + "media_type": "bokeh" + }, + { + "title": "Tax Credits by Wage (Holding Other Inputs Constant)", + "filename": "Tax Credits by Wage (Holding Other Inputs Constant).json", + "media_type": "bokeh" + }, + { + "title": "Calculation of Liabilities", + "filename": "Calculation of Liabilities.html", + "media_type": "table" + } + ], + "ziplocation": "c497e735-5653-417b-ae45-cbccc8be658f_renderable.zip" + }, + "downloadable": { + "outputs": [ + { + "title": "basic_table", + "filename": "basic_table.csv", + "media_type": "CSV" + }, + { + "title": "calculation_table", + "filename": "calculation_table.csv", + "media_type": "CSV" + } + ], + "ziplocation": "c497e735-5653-417b-ae45-cbccc8be658f_downloadable.zip" + } + }, + "version": "v1" +} \ No newline at end of file diff --git a/cs_storage/tests/test_screenshot.py b/cs_storage/tests/test_screenshot.py new file mode 100644 index 0000000..da62733 --- /dev/null +++ b/cs_storage/tests/test_screenshot.py @@ -0,0 +1,29 @@ +import json +import os + +import cs_storage + + +CURRENT_DIR = os.path.abspath(os.path.dirname(__file__)) + + +def test_taxcruncher_outputs(): + with open(f"{CURRENT_DIR}/test-tc-remote.json") as f: + remote_outputs = json.loads(f.read()) + outputs = cs_storage.read(remote_outputs["outputs"]) + + for output in outputs["renderable"]: + basename = f"{output['title'] or 'template'}.html" + print(f"screenshotting: {basename}") + cs_storage.screenshot(output) + + +def test_taxbrain_outputs(): + with open(f"{CURRENT_DIR}/test-tb-remote.json") as f: + remote_outputs = json.loads(f.read()) + outputs = cs_storage.read(remote_outputs["outputs"]) + + for output in outputs["renderable"]: + basename = f"{output['title'] or 'template'}.html" + print(f"screenshotting: {basename}") + cs_storage.screenshot(output) diff --git a/environment.yml b/environment.yml index 4abf0a1..9ef256a 100644 --- a/environment.yml +++ b/environment.yml @@ -5,3 +5,6 @@ dependencies: - "marshmallow>=3.0.0" - pytest - gcsfs + - jinja2 # optional + - pyppeteer # optional + - websockets==6.0 \ No newline at end of file From f3619146cb31c07c63256f95113422ed9921b7eb Mon Sep 17 00:00:00 2001 From: hdoupe Date: Wed, 18 Sep 2019 15:40:26 -0400 Subject: [PATCH 02/18] Use screenshot when writing data --- cs_storage/__init__.py | 38 ++++++++++++++-- cs_storage/screenshot.py | 19 ++++---- cs_storage/tests/test_cs_storage.py | 68 ++++++++++------------------- 3 files changed, 68 insertions(+), 57 deletions(-) diff --git a/cs_storage/__init__.py b/cs_storage/__init__.py index 136f91f..6194ed5 100644 --- a/cs_storage/__init__.py +++ b/cs_storage/__init__.py @@ -10,7 +10,7 @@ from marshmallow import Schema, fields, validate -from .screenshot import screenshot +from .screenshot import screenshot, ScreenshotError, SCREENSHOT_ENABLED __version__ = "1.6.0" @@ -63,10 +63,25 @@ def get_serializer(media_type): class Output: """Output mixin shared among LocalOutput and RemoteOutput""" + id = fields.UUID() title = fields.Str() media_type = fields.Str( - validate=validate.OneOf(choices=["bokeh", "table", "CSV", "PNG", "JPEG", "MP3", "MP4", "HDF5", "PDF", "Markdown", "Text"]) + validate=validate.OneOf( + choices=[ + "bokeh", + "table", + "CSV", + "PNG", + "JPEG", + "MP3", + "MP4", + "HDF5", + "PDF", + "Markdown", + "Text", + ] + ) ) @@ -98,6 +113,18 @@ class LocalResult(Schema): downloadable = fields.Nested(LocalOutput, many=True) +def write_pic(fs, output): + if SCREENSHOT_ENABLED: + try: + pic_data = screenshot(output) + except ScreenshotError: + print("failed to create screenshot for ", output["id"]) + return + else: + with fs.open(f"{BUCKET}/{output['id']}.png", "wb") as f: + f.write(pic_data) + + def write(task_id, loc_result, do_upload=True): fs = gcsfs.GCSFileSystem() s = time.time() @@ -111,18 +138,21 @@ def write(task_id, loc_result, do_upload=True): for output in loc_result[category]: serializer = get_serializer(output["media_type"]) ser = serializer.serialize(output["data"]) + output_id = str(uuid.uuid4()) filename = output["title"] if not filename.endswith(f".{serializer.ext}"): filename += f".{serializer.ext}" zipfileobj.writestr(filename, ser) rem_result[category]["outputs"].append( { - # "id": uuid.uuid4(), + "id": output_id, "title": output["title"], "media_type": output["media_type"], "filename": filename, } ) + if do_upload and category == "renderable": + write_pic(fs, output) zipfileobj.close() buff.seek(0) if do_upload: @@ -151,7 +181,7 @@ def read(rem_result): rem_data = ser.deserialize(zipfileobj.read(rem_output["filename"])) read[category].append( { - # "id": rem_output["unique_id"], + "id": rem_output.get("id", None), "title": rem_output["title"], "media_type": rem_output["media_type"], "data": rem_data, diff --git a/cs_storage/screenshot.py b/cs_storage/screenshot.py index 1c73d38..cd57020 100644 --- a/cs_storage/screenshot.py +++ b/cs_storage/screenshot.py @@ -8,9 +8,14 @@ from jinja2 import Template from pyppeteer import launch from bokeh.resources import CDN - BASE_ARGS = {"bokeh_scripts": {"cdn_js": CDN.js_files[0], "widget_js": CDN.js_files[1]}} + + BASE_ARGS = { + "bokeh_scripts": {"cdn_js": CDN.js_files[0], "widget_js": CDN.js_files[1]} + } + SCREENSHOT_ENABLED = True except ImportError: + SCREENSHOT_ENABLED = False Template = None launch = None CDN = None @@ -19,8 +24,6 @@ import cs_storage - - CURRENT_DIR = os.path.abspath(os.path.dirname(__file__)) @@ -29,7 +32,7 @@ class ScreenshotError(Exception): def get_template(): - if Template is None: + if not SCREENSHOT_ENABLED: return None with open(f"{CURRENT_DIR}/templates/index.html", "r") as f: text = f.read() @@ -51,7 +54,7 @@ async def _screenshot(template_path, pic_path): """ Use pyppeteer, a python port of puppeteer, to open the template at template_path and take a screenshot of the - output that is rendered within it. + output that is rendered within it. The output is rendered within a Bootstrap card element. This element is only as big as the elements that it contains. @@ -59,7 +62,7 @@ async def _screenshot(template_path, pic_path): card to figure out which part of the screen we need to use for the screenshot! - Note: pyppetter looks stale. If it continues to not be + Note: pyppetter looks stale. If it continues to not be maintained well, then the extremely active, well-maintained puppeteer should be used for creating these screenshots. The downside of using puppeteer is that it is written in nodejs. @@ -85,11 +88,11 @@ async def _screenshot(template_path, pic_path): def screenshot(output): """ - Create screenshot of outputs. The intermediate results are + Create screenshot of outputs. The intermediate results are written to temporary files and a picture, represented as a stream of bytes, is returned. """ - if launch is None: + if not SCREENSHOT_ENABLED: return None html = write_template(output) with tempfile.NamedTemporaryFile(suffix=".html") as temp: diff --git a/cs_storage/tests/test_cs_storage.py b/cs_storage/tests/test_cs_storage.py index 51500f3..30b96a6 100644 --- a/cs_storage/tests/test_cs_storage.py +++ b/cs_storage/tests/test_cs_storage.py @@ -60,68 +60,46 @@ def test_cs_storage(): "title": "bokeh plot", "data": {"html": "
", "javascript": "console.log('hello world')"}, }, - { - "media_type": "table", - "title": "table stuff", - "data": "", - }, - { - "media_type": "PNG", - "title": "PNG data", - "data": b"PNG bytes", - }, - { - "media_type": "JPEG", - "title": "JPEG data", - "data": b"JPEG bytes", - }, - { - "media_type": "MP3", - "title": "MP3 data", - "data": b"MP3 bytes", - }, - - { - "media_type": "MP4", - "title": "MP4 data", - "data": b"MP4 bytes", - }, + {"media_type": "table", "title": "table stuff", "data": "
"}, + {"media_type": "PNG", "title": "PNG data", "data": b"PNG bytes"}, + {"media_type": "JPEG", "title": "JPEG data", "data": b"JPEG bytes"}, + {"media_type": "MP3", "title": "MP3 data", "data": b"MP3 bytes"}, + {"media_type": "MP4", "title": "MP4 data", "data": b"MP4 bytes"}, ], "downloadable": [ - { - "media_type": "CSV", - "title": "CSV file", - "data": "comma,sep,values\n" - }, + {"media_type": "CSV", "title": "CSV file", "data": "comma,sep,values\n"}, { "media_type": "HDF5", "title": "HDF5 file", - "data": b"serialized numpy arrays and such\n" - }, - { - "media_type": "PDF", - "title": "PDF file", - "data": b"some pdf like data." + "data": b"serialized numpy arrays and such\n", }, + {"media_type": "PDF", "title": "PDF file", "data": b"some pdf like data."}, { "media_type": "Markdown", "title": "Markdown file", - "data": "**hello world**" - }, - { - "media_type": "Text", - "title": "Text file", - "data": "text data" + "data": "**hello world**", }, + {"media_type": "Text", "title": "Text file", "data": "text data"}, ], } task_id = uuid.uuid4() rem_res = cs_storage.write(task_id, exp_loc_res) loc_res = cs_storage.read(rem_res) - assert loc_res == exp_loc_res + for output_type in ["renderable", "downloadable"]: + exp_res = exp_loc_res[output_type] + loc_res_without_id = [ + {k: v for k, v in output.items() if k != "id"} + for output in loc_res[output_type] + ] + assert exp_res == loc_res_without_id loc_res1 = cs_storage.read({"renderable": rem_res["renderable"]}) - assert loc_res1["renderable"] == exp_loc_res["renderable"] + exp_res = exp_loc_res["renderable"] + loc_res_without_id = [ + {k: v for k, v in output.items() if k != "id"} + for output in loc_res1["renderable"] + ] + assert exp_res == loc_res_without_id def test_errors(): From 9bbdb2ddd20b0b2c907bcc6e48f916dc8c026a6c Mon Sep 17 00:00:00 2001 From: hdoupe Date: Wed, 18 Sep 2019 15:41:20 -0400 Subject: [PATCH 03/18] Fix trailing spaces --- cs_storage/screenshot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cs_storage/screenshot.py b/cs_storage/screenshot.py index cd57020..bf625f9 100644 --- a/cs_storage/screenshot.py +++ b/cs_storage/screenshot.py @@ -54,7 +54,7 @@ async def _screenshot(template_path, pic_path): """ Use pyppeteer, a python port of puppeteer, to open the template at template_path and take a screenshot of the - output that is rendered within it. + output that is rendered within it. The output is rendered within a Bootstrap card element. This element is only as big as the elements that it contains. @@ -62,7 +62,7 @@ async def _screenshot(template_path, pic_path): card to figure out which part of the screen we need to use for the screenshot! - Note: pyppetter looks stale. If it continues to not be + Note: pyppetter looks stale. If it continues to not be maintained well, then the extremely active, well-maintained puppeteer should be used for creating these screenshots. The downside of using puppeteer is that it is written in nodejs. @@ -88,7 +88,7 @@ async def _screenshot(template_path, pic_path): def screenshot(output): """ - Create screenshot of outputs. The intermediate results are + Create screenshot of outputs. The intermediate results are written to temporary files and a picture, represented as a stream of bytes, is returned. """ From 0e4a39d519b86f005d1513ee032ca00991e919a2 Mon Sep 17 00:00:00 2001 From: hdoupe Date: Wed, 25 Sep 2019 09:54:48 -0400 Subject: [PATCH 04/18] Add time info --- cs_storage/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cs_storage/__init__.py b/cs_storage/__init__.py index 6194ed5..b304651 100644 --- a/cs_storage/__init__.py +++ b/cs_storage/__init__.py @@ -115,6 +115,7 @@ class LocalResult(Schema): def write_pic(fs, output): if SCREENSHOT_ENABLED: + s = time.time() try: pic_data = screenshot(output) except ScreenshotError: @@ -123,6 +124,8 @@ def write_pic(fs, output): else: with fs.open(f"{BUCKET}/{output['id']}.png", "wb") as f: f.write(pic_data) + f = time.time() + print(f"Pic write finished in {f-s}s") def write(task_id, loc_result, do_upload=True): From 0fb56288bae95e31b625ac0e6616346d152afed0 Mon Sep 17 00:00:00 2001 From: hdoupe Date: Wed, 25 Sep 2019 17:19:22 -0400 Subject: [PATCH 05/18] Add function for inserting screenshot links for each output --- cs_storage/__init__.py | 6 ++++++ cs_storage/tests/test_cs_storage.py | 17 +++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/cs_storage/__init__.py b/cs_storage/__init__.py index b304651..cd703e6 100644 --- a/cs_storage/__init__.py +++ b/cs_storage/__init__.py @@ -193,3 +193,9 @@ def read(rem_result): f = time.time() print(f"Read finished in {f-s}s") return read + + +def add_screenshot_links(rem_result): + for rem_output in rem_result["renderable"]["outputs"]: + rem_output["screenshot"] = f"https://storage.cloud.google.com/{BUCKET}/{rem_output['id']}.png" + return rem_result \ No newline at end of file diff --git a/cs_storage/tests/test_cs_storage.py b/cs_storage/tests/test_cs_storage.py index 30b96a6..d1f41eb 100644 --- a/cs_storage/tests/test_cs_storage.py +++ b/cs_storage/tests/test_cs_storage.py @@ -102,6 +102,23 @@ def test_cs_storage(): assert exp_res == loc_res_without_id +def test_add_screenshot_links(): + rem_res = { + "renderable": { + "outputs": [{"id": "1234"}, {"id": "4567"}] + } + } + + url = f"https://storage.cloud.google.com/{cs_storage.BUCKET}/" + assert cs_storage.add_screenshot_links(rem_res) == { + "renderable": { + "outputs": [ + {"id": "1234", "screenshot": url + "1234.png"}, + {"id": "4567", "screenshot": url + "4567.png"} + ] + } + } + def test_errors(): with pytest.raises(exceptions.ValidationError): cs_storage.write("123", {"bad": "data"}) From b6649952889f7d929628f86d6579cd954884475f Mon Sep 17 00:00:00 2001 From: hdoupe Date: Fri, 8 Nov 2019 17:43:29 -0500 Subject: [PATCH 06/18] Use dev data for tests --- cs_storage/tests/test-tb-remote.json | 520 +-------------------------- cs_storage/tests/test-tc-remote.json | 11 +- 2 files changed, 27 insertions(+), 504 deletions(-) diff --git a/cs_storage/tests/test-tb-remote.json b/cs_storage/tests/test-tb-remote.json index 142d8bf..795c223 100644 --- a/cs_storage/tests/test-tb-remote.json +++ b/cs_storage/tests/test-tb-remote.json @@ -3,592 +3,108 @@ "renderable": { "outputs": [ { + "id": "ce297163-20cc-44a3-b565-9fa57bcef8ae", "title": "", "filename": ".json", "media_type": "bokeh" }, { + "id": "66c7e9df-62f5-4199-a137-168bd32d245a", "title": "Aggregate Results", "filename": "Aggregate Results.json", "media_type": "bokeh" }, { + "id": "5788882a-8c5f-4a90-bb40-1b1b24d72667", "title": "Tables", "filename": "Tables.json", "media_type": "bokeh" } ], - "ziplocation": "16fd7144-08f7-4b77-b239-8012fa1fa8b5_renderable.zip" + "ziplocation": "0cde3e26-94ea-4483-814a-3aa397d9d5e6_renderable.zip" }, "downloadable": { "outputs": [ { + "id": "60d0f4b0-d687-4a1a-8ac3-d4cf44ec6158", "title": "Total Liabilities Change by Calendar Year (Billions).csv", "filename": "Total Liabilities Change by Calendar Year (Billions).csv", "media_type": "CSV" }, { + "id": "92b7cb97-5086-4074-bab8-6936a24154ec", "title": "Total Liabilities Baseline by Calendar Year (Billions).csv", "filename": "Total Liabilities Baseline by Calendar Year (Billions).csv", "media_type": "CSV" }, { + "id": "9549c414-92bb-463a-83cd-0f20f94cd98c", "title": "Total Liabilities Reform by Calendar Year (Billions).csv", "filename": "Total Liabilities Reform by Calendar Year (Billions).csv", "media_type": "CSV" }, { + "id": "81cb43e9-b906-466f-83c6-7321a7372d8e", "title": "Base plan tax vars, weighted total by expanded income bin (2019).csv", "filename": "Base plan tax vars, weighted total by expanded income bin (2019).csv", "media_type": "CSV" }, { - "title": "Base plan tax vars, weighted total by expanded income bin (2020).csv", - "filename": "Base plan tax vars, weighted total by expanded income bin (2020).csv", - "media_type": "CSV" - }, - { - "title": "Base plan tax vars, weighted total by expanded income bin (2021).csv", - "filename": "Base plan tax vars, weighted total by expanded income bin (2021).csv", - "media_type": "CSV" - }, - { - "title": "Base plan tax vars, weighted total by expanded income bin (2022).csv", - "filename": "Base plan tax vars, weighted total by expanded income bin (2022).csv", - "media_type": "CSV" - }, - { - "title": "Base plan tax vars, weighted total by expanded income bin (2023).csv", - "filename": "Base plan tax vars, weighted total by expanded income bin (2023).csv", - "media_type": "CSV" - }, - { - "title": "Base plan tax vars, weighted total by expanded income bin (2024).csv", - "filename": "Base plan tax vars, weighted total by expanded income bin (2024).csv", - "media_type": "CSV" - }, - { - "title": "Base plan tax vars, weighted total by expanded income bin (2025).csv", - "filename": "Base plan tax vars, weighted total by expanded income bin (2025).csv", - "media_type": "CSV" - }, - { - "title": "Base plan tax vars, weighted total by expanded income bin (2026).csv", - "filename": "Base plan tax vars, weighted total by expanded income bin (2026).csv", - "media_type": "CSV" - }, - { - "title": "Base plan tax vars, weighted total by expanded income bin (2027).csv", - "filename": "Base plan tax vars, weighted total by expanded income bin (2027).csv", - "media_type": "CSV" - }, - { - "title": "Base plan tax vars, weighted total by expanded income bin (2028).csv", - "filename": "Base plan tax vars, weighted total by expanded income bin (2028).csv", - "media_type": "CSV" - }, - { - "title": "Base plan tax vars, weighted total by expanded income bin (2029).csv", - "filename": "Base plan tax vars, weighted total by expanded income bin (2029).csv", - "media_type": "CSV" - }, - { + "id": "2a3387d8-3837-419f-bf9f-210e4883f7ea", "title": "User plan tax vars, weighted total by expanded income bin (2019).csv", "filename": "User plan tax vars, weighted total by expanded income bin (2019).csv", "media_type": "CSV" }, { - "title": "User plan tax vars, weighted total by expanded income bin (2020).csv", - "filename": "User plan tax vars, weighted total by expanded income bin (2020).csv", - "media_type": "CSV" - }, - { - "title": "User plan tax vars, weighted total by expanded income bin (2021).csv", - "filename": "User plan tax vars, weighted total by expanded income bin (2021).csv", - "media_type": "CSV" - }, - { - "title": "User plan tax vars, weighted total by expanded income bin (2022).csv", - "filename": "User plan tax vars, weighted total by expanded income bin (2022).csv", - "media_type": "CSV" - }, - { - "title": "User plan tax vars, weighted total by expanded income bin (2023).csv", - "filename": "User plan tax vars, weighted total by expanded income bin (2023).csv", - "media_type": "CSV" - }, - { - "title": "User plan tax vars, weighted total by expanded income bin (2024).csv", - "filename": "User plan tax vars, weighted total by expanded income bin (2024).csv", - "media_type": "CSV" - }, - { - "title": "User plan tax vars, weighted total by expanded income bin (2025).csv", - "filename": "User plan tax vars, weighted total by expanded income bin (2025).csv", - "media_type": "CSV" - }, - { - "title": "User plan tax vars, weighted total by expanded income bin (2026).csv", - "filename": "User plan tax vars, weighted total by expanded income bin (2026).csv", - "media_type": "CSV" - }, - { - "title": "User plan tax vars, weighted total by expanded income bin (2027).csv", - "filename": "User plan tax vars, weighted total by expanded income bin (2027).csv", - "media_type": "CSV" - }, - { - "title": "User plan tax vars, weighted total by expanded income bin (2028).csv", - "filename": "User plan tax vars, weighted total by expanded income bin (2028).csv", - "media_type": "CSV" - }, - { - "title": "User plan tax vars, weighted total by expanded income bin (2029).csv", - "filename": "User plan tax vars, weighted total by expanded income bin (2029).csv", - "media_type": "CSV" - }, - { + "id": "b40043b0-79e6-4654-9ca9-ea3235421930", "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2019).csv", "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2019).csv", "media_type": "CSV" }, { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2020).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2020).csv", - "media_type": "CSV" - }, - { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2021).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2021).csv", - "media_type": "CSV" - }, - { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2022).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2022).csv", - "media_type": "CSV" - }, - { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2023).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2023).csv", - "media_type": "CSV" - }, - { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2024).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2024).csv", - "media_type": "CSV" - }, - { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2025).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2025).csv", - "media_type": "CSV" - }, - { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2026).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2026).csv", - "media_type": "CSV" - }, - { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2027).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2027).csv", - "media_type": "CSV" - }, - { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2028).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2028).csv", - "media_type": "CSV" - }, - { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2029).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income bin (2029).csv", - "media_type": "CSV" - }, - { + "id": "b8c3cafa-5e23-4cc1-b41a-11740718a14c", "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2019).csv", "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2019).csv", "media_type": "CSV" }, { - "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2020).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2020).csv", - "media_type": "CSV" - }, - { - "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2021).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2021).csv", - "media_type": "CSV" - }, - { - "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2022).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2022).csv", - "media_type": "CSV" - }, - { - "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2023).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2023).csv", - "media_type": "CSV" - }, - { - "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2024).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2024).csv", - "media_type": "CSV" - }, - { - "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2025).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2025).csv", - "media_type": "CSV" - }, - { - "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2026).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2026).csv", - "media_type": "CSV" - }, - { - "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2027).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2027).csv", - "media_type": "CSV" - }, - { - "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2028).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2028).csv", - "media_type": "CSV" - }, - { - "title": "Payroll Tax: Difference between Base and User plans by expanded income bin (2029).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income bin (2029).csv", - "media_type": "CSV" - }, - { + "id": "cec78b20-4137-4b71-8ada-f415791e88ae", "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2019).csv", "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2019).csv", "media_type": "CSV" }, { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2020).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2020).csv", - "media_type": "CSV" - }, - { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2021).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2021).csv", - "media_type": "CSV" - }, - { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2022).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2022).csv", - "media_type": "CSV" - }, - { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2023).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2023).csv", - "media_type": "CSV" - }, - { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2024).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2024).csv", - "media_type": "CSV" - }, - { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2025).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2025).csv", - "media_type": "CSV" - }, - { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2026).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2026).csv", - "media_type": "CSV" - }, - { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2027).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2027).csv", - "media_type": "CSV" - }, - { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2028).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2028).csv", - "media_type": "CSV" - }, - { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2029).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income bin (2029).csv", - "media_type": "CSV" - }, - { + "id": "47517e7d-e033-474b-9b69-97b6c19825fc", "title": "Base plan tax vars, weighted total by expanded income decile (2019).csv", "filename": "Base plan tax vars, weighted total by expanded income decile (2019).csv", "media_type": "CSV" }, { - "title": "Base plan tax vars, weighted total by expanded income decile (2020).csv", - "filename": "Base plan tax vars, weighted total by expanded income decile (2020).csv", - "media_type": "CSV" - }, - { - "title": "Base plan tax vars, weighted total by expanded income decile (2021).csv", - "filename": "Base plan tax vars, weighted total by expanded income decile (2021).csv", - "media_type": "CSV" - }, - { - "title": "Base plan tax vars, weighted total by expanded income decile (2022).csv", - "filename": "Base plan tax vars, weighted total by expanded income decile (2022).csv", - "media_type": "CSV" - }, - { - "title": "Base plan tax vars, weighted total by expanded income decile (2023).csv", - "filename": "Base plan tax vars, weighted total by expanded income decile (2023).csv", - "media_type": "CSV" - }, - { - "title": "Base plan tax vars, weighted total by expanded income decile (2024).csv", - "filename": "Base plan tax vars, weighted total by expanded income decile (2024).csv", - "media_type": "CSV" - }, - { - "title": "Base plan tax vars, weighted total by expanded income decile (2025).csv", - "filename": "Base plan tax vars, weighted total by expanded income decile (2025).csv", - "media_type": "CSV" - }, - { - "title": "Base plan tax vars, weighted total by expanded income decile (2026).csv", - "filename": "Base plan tax vars, weighted total by expanded income decile (2026).csv", - "media_type": "CSV" - }, - { - "title": "Base plan tax vars, weighted total by expanded income decile (2027).csv", - "filename": "Base plan tax vars, weighted total by expanded income decile (2027).csv", - "media_type": "CSV" - }, - { - "title": "Base plan tax vars, weighted total by expanded income decile (2028).csv", - "filename": "Base plan tax vars, weighted total by expanded income decile (2028).csv", - "media_type": "CSV" - }, - { - "title": "Base plan tax vars, weighted total by expanded income decile (2029).csv", - "filename": "Base plan tax vars, weighted total by expanded income decile (2029).csv", - "media_type": "CSV" - }, - { + "id": "49122b30-0baf-4347-bfd1-77d4a1606dae", "title": "User plan tax vars, weighted total by expanded income decile (2019).csv", "filename": "User plan tax vars, weighted total by expanded income decile (2019).csv", "media_type": "CSV" }, { - "title": "User plan tax vars, weighted total by expanded income decile (2020).csv", - "filename": "User plan tax vars, weighted total by expanded income decile (2020).csv", - "media_type": "CSV" - }, - { - "title": "User plan tax vars, weighted total by expanded income decile (2021).csv", - "filename": "User plan tax vars, weighted total by expanded income decile (2021).csv", - "media_type": "CSV" - }, - { - "title": "User plan tax vars, weighted total by expanded income decile (2022).csv", - "filename": "User plan tax vars, weighted total by expanded income decile (2022).csv", - "media_type": "CSV" - }, - { - "title": "User plan tax vars, weighted total by expanded income decile (2023).csv", - "filename": "User plan tax vars, weighted total by expanded income decile (2023).csv", - "media_type": "CSV" - }, - { - "title": "User plan tax vars, weighted total by expanded income decile (2024).csv", - "filename": "User plan tax vars, weighted total by expanded income decile (2024).csv", - "media_type": "CSV" - }, - { - "title": "User plan tax vars, weighted total by expanded income decile (2025).csv", - "filename": "User plan tax vars, weighted total by expanded income decile (2025).csv", - "media_type": "CSV" - }, - { - "title": "User plan tax vars, weighted total by expanded income decile (2026).csv", - "filename": "User plan tax vars, weighted total by expanded income decile (2026).csv", - "media_type": "CSV" - }, - { - "title": "User plan tax vars, weighted total by expanded income decile (2027).csv", - "filename": "User plan tax vars, weighted total by expanded income decile (2027).csv", - "media_type": "CSV" - }, - { - "title": "User plan tax vars, weighted total by expanded income decile (2028).csv", - "filename": "User plan tax vars, weighted total by expanded income decile (2028).csv", - "media_type": "CSV" - }, - { - "title": "User plan tax vars, weighted total by expanded income decile (2029).csv", - "filename": "User plan tax vars, weighted total by expanded income decile (2029).csv", - "media_type": "CSV" - }, - { + "id": "e2bd81b6-dccc-4df9-b0fe-ab4bbf1297f8", "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2019).csv", "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2019).csv", "media_type": "CSV" }, { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2020).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2020).csv", - "media_type": "CSV" - }, - { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2021).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2021).csv", - "media_type": "CSV" - }, - { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2022).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2022).csv", - "media_type": "CSV" - }, - { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2023).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2023).csv", - "media_type": "CSV" - }, - { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2024).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2024).csv", - "media_type": "CSV" - }, - { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2025).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2025).csv", - "media_type": "CSV" - }, - { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2026).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2026).csv", - "media_type": "CSV" - }, - { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2027).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2027).csv", - "media_type": "CSV" - }, - { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2028).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2028).csv", - "media_type": "CSV" - }, - { - "title": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2029).csv", - "filename": "Individual Income Tax: Difference between Base and User plans by expanded income decile (2029).csv", - "media_type": "CSV" - }, - { + "id": "047baaff-f2a8-4b8f-b0de-be42ddbe9249", "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2019).csv", "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2019).csv", "media_type": "CSV" }, { - "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2020).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2020).csv", - "media_type": "CSV" - }, - { - "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2021).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2021).csv", - "media_type": "CSV" - }, - { - "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2022).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2022).csv", - "media_type": "CSV" - }, - { - "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2023).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2023).csv", - "media_type": "CSV" - }, - { - "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2024).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2024).csv", - "media_type": "CSV" - }, - { - "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2025).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2025).csv", - "media_type": "CSV" - }, - { - "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2026).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2026).csv", - "media_type": "CSV" - }, - { - "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2027).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2027).csv", - "media_type": "CSV" - }, - { - "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2028).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2028).csv", - "media_type": "CSV" - }, - { - "title": "Payroll Tax: Difference between Base and User plans by expanded income decile (2029).csv", - "filename": "Payroll Tax: Difference between Base and User plans by expanded income decile (2029).csv", - "media_type": "CSV" - }, - { + "id": "9f863037-390b-46c8-8971-ea22afcee630", "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2019).csv", "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2019).csv", "media_type": "CSV" - }, - { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2020).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2020).csv", - "media_type": "CSV" - }, - { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2021).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2021).csv", - "media_type": "CSV" - }, - { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2022).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2022).csv", - "media_type": "CSV" - }, - { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2023).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2023).csv", - "media_type": "CSV" - }, - { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2024).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2024).csv", - "media_type": "CSV" - }, - { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2025).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2025).csv", - "media_type": "CSV" - }, - { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2026).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2026).csv", - "media_type": "CSV" - }, - { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2027).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2027).csv", - "media_type": "CSV" - }, - { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2028).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2028).csv", - "media_type": "CSV" - }, - { - "title": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2029).csv", - "filename": "Combined Payroll and Individual Income Tax: Difference between Base and User plans by expanded income decile (2029).csv", - "media_type": "CSV" } ], - "ziplocation": "16fd7144-08f7-4b77-b239-8012fa1fa8b5_downloadable.zip" + "ziplocation": "0cde3e26-94ea-4483-814a-3aa397d9d5e6_downloadable.zip" } }, "version": "v1" diff --git a/cs_storage/tests/test-tc-remote.json b/cs_storage/tests/test-tc-remote.json index 2e648a2..3913a8d 100644 --- a/cs_storage/tests/test-tc-remote.json +++ b/cs_storage/tests/test-tc-remote.json @@ -3,47 +3,54 @@ "renderable": { "outputs": [ { + "id": "0469459a-82ab-4018-be30-3862a5383d4e", "title": "Basic Liabilities", "filename": "Basic Liabilities.html", "media_type": "table" }, { + "id": "17162a2f-8951-4c8e-9795-44284e520f72", "title": "Tax Liabilities by Wage (Holding Other Inputs Constant)", "filename": "Tax Liabilities by Wage (Holding Other Inputs Constant).json", "media_type": "bokeh" }, { + "id": "43b8914c-6832-4308-b4fa-6a13f899f239", "title": "Tax Rates by Wage (Holding Other Inputs Constant)", "filename": "Tax Rates by Wage (Holding Other Inputs Constant).json", "media_type": "bokeh" }, { + "id": "51dccf29-f9b0-42a8-8c76-17ef6544ba47", "title": "Tax Credits by Wage (Holding Other Inputs Constant)", "filename": "Tax Credits by Wage (Holding Other Inputs Constant).json", "media_type": "bokeh" }, { + "id": "435e4de6-23b4-4693-8a3a-2b7917190f09", "title": "Calculation of Liabilities", "filename": "Calculation of Liabilities.html", "media_type": "table" } ], - "ziplocation": "c497e735-5653-417b-ae45-cbccc8be658f_renderable.zip" + "ziplocation": "0fe89f27-37f3-4342-95a6-ffa952e3d4c6_renderable.zip" }, "downloadable": { "outputs": [ { + "id": "8d34d210-f275-495b-839e-18acc1fc6d2a", "title": "basic_table", "filename": "basic_table.csv", "media_type": "CSV" }, { + "id": "e1b80703-2088-4938-afa4-c3b49bb3142a", "title": "calculation_table", "filename": "calculation_table.csv", "media_type": "CSV" } ], - "ziplocation": "c497e735-5653-417b-ae45-cbccc8be658f_downloadable.zip" + "ziplocation": "0fe89f27-37f3-4342-95a6-ffa952e3d4c6_downloadable.zip" } }, "version": "v1" From 275b4c5b1167ca2ca9f20834d0b69d8fad715bed Mon Sep 17 00:00:00 2001 From: hdoupe Date: Tue, 12 Nov 2019 11:34:23 -0500 Subject: [PATCH 07/18] Add bokeh to environment.yaml --- environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 9ef256a..14ef2a8 100644 --- a/environment.yml +++ b/environment.yml @@ -7,4 +7,5 @@ dependencies: - gcsfs - jinja2 # optional - pyppeteer # optional - - websockets==6.0 \ No newline at end of file + - bokeh # optional + - websockets==6.0 # optional \ No newline at end of file From eeddad9a6782eb3d949e795c17b6b09a3b6e437c Mon Sep 17 00:00:00 2001 From: hdoupe Date: Tue, 12 Nov 2019 11:34:56 -0500 Subject: [PATCH 08/18] Print warning when write_pic is called without necessary packages --- cs_storage/__init__.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/cs_storage/__init__.py b/cs_storage/__init__.py index cd703e6..6e719d2 100644 --- a/cs_storage/__init__.py +++ b/cs_storage/__init__.py @@ -126,6 +126,13 @@ def write_pic(fs, output): f.write(pic_data) f = time.time() print(f"Pic write finished in {f-s}s") + else: + import warnings + + warnings.warn( + "Screenshot not enabled. Make sure you have installed " + "the optional packages listed in environment.yaml." + ) def write(task_id, loc_result, do_upload=True): @@ -197,5 +204,7 @@ def read(rem_result): def add_screenshot_links(rem_result): for rem_output in rem_result["renderable"]["outputs"]: - rem_output["screenshot"] = f"https://storage.cloud.google.com/{BUCKET}/{rem_output['id']}.png" - return rem_result \ No newline at end of file + rem_output[ + "screenshot" + ] = f"https://storage.cloud.google.com/{BUCKET}/{rem_output['id']}.png" + return rem_result From 4fea7842bf1df49c951efc6699c16caf5524f3dc Mon Sep 17 00:00:00 2001 From: hdoupe Date: Tue, 12 Nov 2019 16:44:16 -0500 Subject: [PATCH 09/18] Add PNG and JPEG to screenshot template --- cs_storage/templates/index.html | 40 +++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/cs_storage/templates/index.html b/cs_storage/templates/index.html index d25327e..3e0ddd7 100644 --- a/cs_storage/templates/index.html +++ b/cs_storage/templates/index.html @@ -1,23 +1,30 @@ + - Compute Studio - - - - - - + Compute Studio + + + + + + - + - - + + - +
@@ -31,9 +38,18 @@

{{output.title}}

{{output.data|safe}}
+ {% elif output.media_type == 'PNG' %} +
+ +
+ {% elif output.media_type == 'JPEG' %} +
+ +
{% endif %}
- + + \ No newline at end of file From ecda2c4ce67c4e474176b679151b8e77c471f1fc Mon Sep 17 00:00:00 2001 From: hdoupe Date: Tue, 12 Nov 2019 17:10:16 -0500 Subject: [PATCH 10/18] Drop websockets==6.0 pin --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 14ef2a8..2372239 100644 --- a/environment.yml +++ b/environment.yml @@ -8,4 +8,4 @@ dependencies: - jinja2 # optional - pyppeteer # optional - bokeh # optional - - websockets==6.0 # optional \ No newline at end of file + - websockets # optional From a03236ca6163f0949c940dd8f573dca0d3741673 Mon Sep 17 00:00:00 2001 From: hdoupe Date: Tue, 12 Nov 2019 17:41:27 -0500 Subject: [PATCH 11/18] Make id optional since cs-storage adds it after the data is loaded --- cs_storage/__init__.py | 6 +++--- cs_storage/tests/test_cs_storage.py | 27 ++++++++++++++++----------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/cs_storage/__init__.py b/cs_storage/__init__.py index 6e719d2..5722060 100644 --- a/cs_storage/__init__.py +++ b/cs_storage/__init__.py @@ -64,7 +64,7 @@ def get_serializer(media_type): class Output: """Output mixin shared among LocalOutput and RemoteOutput""" - id = fields.UUID() + id = fields.UUID(required=False) title = fields.Str() media_type = fields.Str( validate=validate.OneOf( @@ -148,14 +148,14 @@ def write(task_id, loc_result, do_upload=True): for output in loc_result[category]: serializer = get_serializer(output["media_type"]) ser = serializer.serialize(output["data"]) - output_id = str(uuid.uuid4()) + output["id"] = str(uuid.uuid4()) filename = output["title"] if not filename.endswith(f".{serializer.ext}"): filename += f".{serializer.ext}" zipfileobj.writestr(filename, ser) rem_result[category]["outputs"].append( { - "id": output_id, + "id": output["id"], "title": output["title"], "media_type": output["media_type"], "filename": filename, diff --git a/cs_storage/tests/test_cs_storage.py b/cs_storage/tests/test_cs_storage.py index d1f41eb..edc296e 100644 --- a/cs_storage/tests/test_cs_storage.py +++ b/cs_storage/tests/test_cs_storage.py @@ -53,6 +53,7 @@ def test_get_serializer(): def test_cs_storage(): + dummy_uuid = "c7a65ad2-0c2c-45d7-b0f7-d9fd524c49b3" exp_loc_res = { "renderable": [ { @@ -82,43 +83,47 @@ def test_cs_storage(): {"media_type": "Text", "title": "Text file", "data": "text data"}, ], } - task_id = uuid.uuid4() + task_id = "1868c4a7-b03c-4fe4-ab45-0aa95c0bfa53" rem_res = cs_storage.write(task_id, exp_loc_res) loc_res = cs_storage.read(rem_res) for output_type in ["renderable", "downloadable"]: - exp_res = exp_loc_res[output_type] loc_res_without_id = [ {k: v for k, v in output.items() if k != "id"} for output in loc_res[output_type] ] - assert exp_res == loc_res_without_id + exp_res_without_id = [ + {k: v for k, v in output.items() if k != "id"} + for output in exp_loc_res[output_type] + ] + assert exp_res_without_id == loc_res_without_id loc_res1 = cs_storage.read({"renderable": rem_res["renderable"]}) - exp_res = exp_loc_res["renderable"] loc_res_without_id = [ {k: v for k, v in output.items() if k != "id"} for output in loc_res1["renderable"] ] - assert exp_res == loc_res_without_id + exp_res_without_id = [ + {k: v for k, v in output.items() if k != "id"} + for output in exp_loc_res["renderable"] + ] + + assert exp_res_without_id == loc_res_without_id def test_add_screenshot_links(): - rem_res = { - "renderable": { - "outputs": [{"id": "1234"}, {"id": "4567"}] - } - } + rem_res = {"renderable": {"outputs": [{"id": "1234"}, {"id": "4567"}]}} url = f"https://storage.cloud.google.com/{cs_storage.BUCKET}/" assert cs_storage.add_screenshot_links(rem_res) == { "renderable": { "outputs": [ {"id": "1234", "screenshot": url + "1234.png"}, - {"id": "4567", "screenshot": url + "4567.png"} + {"id": "4567", "screenshot": url + "4567.png"}, ] } } + def test_errors(): with pytest.raises(exceptions.ValidationError): cs_storage.write("123", {"bad": "data"}) From c49944fa00a81162e0ddf79e32fa319a432b6548 Mon Sep 17 00:00:00 2001 From: hdoupe Date: Tue, 12 Nov 2019 17:51:38 -0500 Subject: [PATCH 12/18] Add manifest.in so that templates are included --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..8ab20b4 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include cs_config/templates/index.html \ No newline at end of file From de6ac0a111cfa139c1dff3daefe09508f6151623 Mon Sep 17 00:00:00 2001 From: hdoupe Date: Tue, 12 Nov 2019 17:59:09 -0500 Subject: [PATCH 13/18] Use correct directory --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 8ab20b4..5c3bb29 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1 @@ -include cs_config/templates/index.html \ No newline at end of file +include cs_storage/templates/index.html \ No newline at end of file From f20450a7fd1bc0b7b4f49f1f3d7f65bc0f3db6a2 Mon Sep 17 00:00:00 2001 From: hdoupe Date: Wed, 13 Nov 2019 12:14:39 -0500 Subject: [PATCH 14/18] Use --no-sandbox arg so that it can run in docker --- cs_storage/screenshot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cs_storage/screenshot.py b/cs_storage/screenshot.py index bf625f9..49b1c8e 100644 --- a/cs_storage/screenshot.py +++ b/cs_storage/screenshot.py @@ -67,7 +67,7 @@ async def _screenshot(template_path, pic_path): puppeteer should be used for creating these screenshots. The downside of using puppeteer is that it is written in nodejs. """ - browser = await launch() + browser = await launch(args=["--no-sandbox"]) page = await browser.newPage() await page.goto(f"file://{template_path}") await page.setViewport(dict(width=1920, height=1080)) From 38b9286b94a1e81a7336c1e9a319bc4d5032d1b5 Mon Sep 17 00:00:00 2001 From: hdoupe Date: Wed, 13 Nov 2019 14:04:10 -0500 Subject: [PATCH 15/18] Add debug mode for screenshot to view intermediate HTML file --- cs_storage/screenshot.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cs_storage/screenshot.py b/cs_storage/screenshot.py index 49b1c8e..f4c426a 100644 --- a/cs_storage/screenshot.py +++ b/cs_storage/screenshot.py @@ -86,7 +86,7 @@ async def _screenshot(template_path, pic_path): await browser.close() -def screenshot(output): +def screenshot(output, debug=False): """ Create screenshot of outputs. The intermediate results are written to temporary files and a picture, represented as a @@ -96,6 +96,9 @@ def screenshot(output): return None html = write_template(output) with tempfile.NamedTemporaryFile(suffix=".html") as temp: + if debug: + with open(f'{output["title"]}.html', "w") as f: + f.write(html) temp.write(html.encode("utf-8")) temp.seek(0) template_path = temp.name From cb91a3e956edd604b17583de1fecc28ca1e7e6c8 Mon Sep 17 00:00:00 2001 From: hdoupe Date: Wed, 13 Nov 2019 14:04:36 -0500 Subject: [PATCH 16/18] Update bokeh output in template --- cs_storage/templates/index.html | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cs_storage/templates/index.html b/cs_storage/templates/index.html index 3e0ddd7..f712ebe 100644 --- a/cs_storage/templates/index.html +++ b/cs_storage/templates/index.html @@ -30,9 +30,8 @@

{{output.title}}

{% if output.media_type == 'bokeh' %} -
- {{output.data.html|safe}} - {{output.data.javascript|safe}} +
{{output.id}} +
{% elif output.media_type == 'table' %}
@@ -52,4 +51,10 @@

{{output.title}}

+{% if output.media_type == "bokeh" %} + +{% endif %} + \ No newline at end of file From 9bae4a167e19b8d944b2dcc4053af6aa47d884e8 Mon Sep 17 00:00:00 2001 From: hdoupe Date: Fri, 3 Jan 2020 15:03:52 -0500 Subject: [PATCH 17/18] Use api endpoint that doesn't ask for auth --- cs_storage/__init__.py | 2 +- cs_storage/tests/test_cs_storage.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cs_storage/__init__.py b/cs_storage/__init__.py index 5722060..0deec69 100644 --- a/cs_storage/__init__.py +++ b/cs_storage/__init__.py @@ -206,5 +206,5 @@ def add_screenshot_links(rem_result): for rem_output in rem_result["renderable"]["outputs"]: rem_output[ "screenshot" - ] = f"https://storage.cloud.google.com/{BUCKET}/{rem_output['id']}.png" + ] = f"https://storage.googleapis.com/{BUCKET}/{rem_output['id']}.png" return rem_result diff --git a/cs_storage/tests/test_cs_storage.py b/cs_storage/tests/test_cs_storage.py index edc296e..5727bd5 100644 --- a/cs_storage/tests/test_cs_storage.py +++ b/cs_storage/tests/test_cs_storage.py @@ -113,7 +113,7 @@ def test_cs_storage(): def test_add_screenshot_links(): rem_res = {"renderable": {"outputs": [{"id": "1234"}, {"id": "4567"}]}} - url = f"https://storage.cloud.google.com/{cs_storage.BUCKET}/" + url = f"https://storage.googleapis.com/{cs_storage.BUCKET}/" assert cs_storage.add_screenshot_links(rem_res) == { "renderable": { "outputs": [ From ed07fc250cb6e72e79b69442ce98ce846bd016d8 Mon Sep 17 00:00:00 2001 From: hdoupe Date: Wed, 8 Jan 2020 12:01:24 -0500 Subject: [PATCH 18/18] Fix merge conflicts --- cs_storage/tests/test_cs_storage.py | 64 ++++++++++++----------------- 1 file changed, 26 insertions(+), 38 deletions(-) diff --git a/cs_storage/tests/test_cs_storage.py b/cs_storage/tests/test_cs_storage.py index c311c9f..bb1f5db 100644 --- a/cs_storage/tests/test_cs_storage.py +++ b/cs_storage/tests/test_cs_storage.py @@ -15,13 +15,14 @@ def png(): import matplotlib.pyplot as plt import numpy as np + x = np.linspace(0, 2, 100) plt.figure() - plt.plot(x, x, label='linear') - plt.plot(x, x**2, label='quadratic') - plt.plot(x, x**3, label='cubic') - plt.xlabel('x label') - plt.ylabel('y label') + plt.plot(x, x, label="linear") + plt.plot(x, x ** 2, label="quadratic") + plt.plot(x, x ** 3, label="cubic") + plt.xlabel("x label") + plt.ylabel("y label") plt.title("Simple Plot") plt.legend() initial_buff = io.BytesIO() @@ -34,13 +35,14 @@ def png(): def jpg(): import matplotlib.pyplot as plt import numpy as np + x = np.linspace(0, 2, 100) plt.figure() - plt.plot(x, x, label='linear') - plt.plot(x, x**2, label='quadratic') - plt.plot(x, x**3, label='cubic') - plt.xlabel('x label') - plt.ylabel('y label') + plt.plot(x, x, label="linear") + plt.plot(x, x ** 2, label="quadratic") + plt.plot(x, x ** 3, label="cubic") + plt.xlabel("x label") + plt.ylabel("y label") plt.title("Simple Plot") plt.legend() initial_buff = io.BytesIO() @@ -115,32 +117,11 @@ def test_cs_storage(png, jpg): "title": "bokeh plot", "data": {"html": "
", "javascript": "console.log('hello world')"}, }, - { - "media_type": "table", - "title": "table stuff", - "data": "
", - }, - { - "media_type": "PNG", - "title": "PNG data", - "data": png, - }, - { - "media_type": "JPEG", - "title": "JPEG data", - "data": jpg, - }, - { - "media_type": "MP3", - "title": "MP3 data", - "data": b"MP3 bytes", - }, - - { - "media_type": "MP4", - "title": "MP4 data", - "data": b"MP4 bytes", - }, + {"media_type": "table", "title": "table stuff", "data": "
"}, + {"media_type": "PNG", "title": "PNG data", "data": png}, + {"media_type": "JPEG", "title": "JPEG data", "data": jpg}, + {"media_type": "MP3", "title": "MP3 data", "data": b"MP3 bytes"}, + {"media_type": "MP4", "title": "MP4 data", "data": b"MP4 bytes"}, ], "downloadable": [ {"media_type": "CSV", "title": "CSV file", "data": "comma,sep,values\n"}, @@ -160,7 +141,7 @@ def test_cs_storage(png, jpg): } task_id = "1868c4a7-b03c-4fe4-ab45-0aa95c0bfa53" rem_res = cs_storage.write(task_id, exp_loc_res) - loc_res = cs_storage.read(rem_res) + loc_res = cs_storage.read(rem_res, json_serializable=False) for output_type in ["renderable", "downloadable"]: loc_res_without_id = [ {k: v for k, v in output.items() if k != "id"} @@ -172,7 +153,11 @@ def test_cs_storage(png, jpg): ] assert exp_res_without_id == loc_res_without_id - loc_res1 = cs_storage.read({"renderable": rem_res["renderable"]}) + assert json.dumps(cs_storage.read(rem_res, json_serializable=True)) + + loc_res1 = cs_storage.read( + {"renderable": rem_res["renderable"]}, json_serializable=False + ) loc_res_without_id = [ {k: v for k, v in output.items() if k != "id"} for output in loc_res1["renderable"] @@ -183,6 +168,9 @@ def test_cs_storage(png, jpg): ] assert exp_res_without_id == loc_res_without_id + assert json.dumps( + cs_storage.read({"renderable": rem_res["renderable"]}, json_serializable=True) + ) def test_add_screenshot_links():