From d14141a0e1e34a1204f0117d7613f6d978606fe0 Mon Sep 17 00:00:00 2001 From: Vladimir Tarbaev Date: Mon, 6 May 2024 23:59:51 +0300 Subject: [PATCH 1/5] setting up multiple versions of docs --- .github/workflows/ci.yml | 36 +++++++++++++++++++++++++++--------- docs/mkdocs.yml | 7 ++++--- docs/overrides/main.html | 9 ++++++++- mkdocs.yml | 7 ++++--- requirements.txt | 1 + 5 files changed, 44 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d25047c3..9ad30f8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,20 +180,38 @@ jobs: - name: Install dependicies run: pip install -r requirements.txt + - name: Setup docs deploy + run: | + git config --global user.name "Docs Deployer" + git config --global user.email docs@greenmask.io + echo "TAGS=$(git log --tags --simplify-by-decoration --pretty='format:%d' --since=2024-04-30 | sed '/^$/d' | cut -d ':' -f 2 | cut -d ')' -f 1 | tac | tr '\n' ' ')" >> $GITHUB_ENV + - name: Build docs - run: mkdocs build + run: | + for tag in ${{ env.TAGS }}; do + echo "### CHECKOUT TO ${tag} ###" + git checkout $tag + if [[ "$tag" == *"rc"* || "$tag" == *"dev"* || "$tag" == *"pre"* ]]; then + mike deploy $tag + else + mike deploy --update-aliases $tag latest + fi + done - - name: Create docs directory - run: sudo mkdir -p ${{ secrets.DOCS_DEPLOY_DIR }} + - name: Set latest version as default + run: mike set-default latest - - name: Move html files to docs directory - run: sudo mv site ${{ secrets.DOCS_DEPLOY_DIR }}/html-${{ github.ref_name}} + - name: Change branch to get html files + run: git checkout gh-pages + + - name: Remove old folder + run: sudo rm -rf ${{ secrets.DOCS_DEPLOY_DIR }}/html - - name: Remove old symlink - run: sudo unlink ${{ secrets.DOCS_DEPLOY_DIR }}/html + - name: Create docs directory + run: sudo mkdir -p ${{ secrets.DOCS_DEPLOY_DIR }}/html - - name: Create new symlink - run: sudo ln -s ${{ secrets.DOCS_DEPLOY_DIR }}/html-${{ github.ref_name}} ${{ secrets.DOCS_DEPLOY_DIR }}/html + - name: Move html files to docs directory + run: sudo cp -r * ${{ secrets.DOCS_DEPLOY_DIR }}/html - name: Restart web service run: sudo systemctl restart nginx diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 6bdd85f3..fed98024 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -134,13 +134,13 @@ nav: repo_url: https://github.com/GreenmaskIO/greenmask repo_name: GreenmaskIO/greenmask -site_url: https://greenmask.io +site_url: https://greenmask.io/ copyright: Copyright © 2024 Greenmask extra: - # version: - # provider: mike + version: + provider: mike social: - icon: fontawesome/brands/x-twitter link: https://twitter.com/GreenmaskIO @@ -152,3 +152,4 @@ extra: plugins: - social - search + - mike diff --git a/docs/overrides/main.html b/docs/overrides/main.html index 8b6265e6..1635f8f5 100644 --- a/docs/overrides/main.html +++ b/docs/overrides/main.html @@ -1,5 +1,12 @@ {% extends "base.html" %} {% block announce %} - Version 0.1.12 is released + Version 0.1.12 is released {% endblock %} + +{% block outdated %} + You're not viewing the latest version. + + Click here to go to latest. + +{% endblock %} \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 8f4b225e..a99fe076 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -134,13 +134,13 @@ nav: repo_url: https://github.com/GreenmaskIO/greenmask repo_name: GreenmaskIO/greenmask -site_url: https://greenmask.io +site_url: https://greenmask.io/ copyright: Copyright © 2024 Greenmask extra: - # version: - # provider: mike + version: + provider: mike social: - icon: fontawesome/brands/x-twitter link: https://twitter.com/GreenmaskIO @@ -152,3 +152,4 @@ extra: plugins: - social - search + - mike diff --git a/requirements.txt b/requirements.txt index 3ffaa604..0ae79d0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,3 +35,4 @@ tinycss2==1.2.1 urllib3==2.2.1 watchdog==4.0.0 webencodings==0.5.1 +mike==2.1.1 From cbfc000cfff7f1a5275cc0bc43f7094182937406 Mon Sep 17 00:00:00 2001 From: Vladimir Tarbaev Date: Tue, 7 May 2024 09:50:57 +0300 Subject: [PATCH 2/5] setting up multiple versions of docs --- docs/mkdocs.yml | 155 ------------------------------------------------ 1 file changed, 155 deletions(-) delete mode 100644 docs/mkdocs.yml diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml deleted file mode 100644 index fed98024..00000000 --- a/docs/mkdocs.yml +++ /dev/null @@ -1,155 +0,0 @@ -site_name: Greenmask — PostgreSQL masking and obfuscation tool - -# Theme -theme: - name: material - custom_dir: docs/overrides - logo: assets/logo.png - favicon: assets/logo.png - features: - - navigation.indexes - - announce.dismiss - - content.code.annotate - - content.code.copy - - content.tooltips - # - navigation.sections - - navigation.tabs - # - navigation.top - # - navigation.tracking - - search.highlight - - search.share - - search.suggest - # - toc.follow - palette: - scheme: default - primary: teal - accent: indigo - toggle: - icon: material/brightness-7 - - -# Markdown extensions -markdown_extensions: - - attr_list - - toc: - permalink: true - - pymdownx.highlight - - pymdownx.superfences - - pymdownx.tabbed: - alternate_style: true - - pymdownx.snippets - - admonition - - pymdownx.details - - tables - -nav: - - Home: - - Home: index.md - - Documentation: - - Installation: installation.md - - Configuration: configuration.md - - Architecture: architecture.md - - Playground: playground.md - - Commands: commands.md - - Transformers: - - built_in_transformers/index.md - - Standard transformers: - - built_in_transformers/standard_transformers/index.md - - Cmd: built_in_transformers/standard_transformers/cmd.md - - Dict: built_in_transformers/standard_transformers/dict.md - - Hash: built_in_transformers/standard_transformers/hash.md - - Masking: built_in_transformers/standard_transformers/masking.md - - NoiseDate: built_in_transformers/standard_transformers/noise_date.md - - NoiseFloat: built_in_transformers/standard_transformers/noise_float.md - - NoiseInt: built_in_transformers/standard_transformers/noise_int.md - - RandomBool: built_in_transformers/standard_transformers/random_bool.md - - RandomChoice: built_in_transformers/standard_transformers/random_choice.md - - RandomDate: built_in_transformers/standard_transformers/random_date.md - - RandomFloat: built_in_transformers/standard_transformers/random_float.md - - RandomInt: built_in_transformers/standard_transformers/random_int.md - - RandomString: built_in_transformers/standard_transformers/random_string.md - - RandomUuid: built_in_transformers/standard_transformers/random_uuid.md - - RandomLatitude: built_in_transformers/standard_transformers/random_latitude.md - - RandomLongitude: built_in_transformers/standard_transformers/random_longitude.md - - RandomUnixTime: built_in_transformers/standard_transformers/random_unix_time.md - - RandomDayOfWeek: built_in_transformers/standard_transformers/random_day_of_week.md - - RandomDayOfMonth: built_in_transformers/standard_transformers/random_day_of_month.md - - RandomMonthName: built_in_transformers/standard_transformers/random_month_name.md - - RandomYearString: built_in_transformers/standard_transformers/random_year_string.md - - RandomCentury: built_in_transformers/standard_transformers/random_century.md - - RandomTimezone: built_in_transformers/standard_transformers/random_timezone.md - - RandomEmail: built_in_transformers/standard_transformers/random_email.md - - RandomUsername: built_in_transformers/standard_transformers/random_username.md - - RandomPassword: built_in_transformers/standard_transformers/random_password.md - - RandomMacAddress: built_in_transformers/standard_transformers/random_mac_address.md - - RandomDomainName: built_in_transformers/standard_transformers/random_domain_name.md - - RandomURL: built_in_transformers/standard_transformers/random_url.md - - RandomIPv4: built_in_transformers/standard_transformers/random_ipv4.md - - RandomIPv6: built_in_transformers/standard_transformers/random_ipv6.md - - RandomWord: built_in_transformers/standard_transformers/random_word.md - - RandomSentence: built_in_transformers/standard_transformers/random_sentence.md - - RandomParagraph: built_in_transformers/standard_transformers/random_paragraph.md - - RandomCCType: built_in_transformers/standard_transformers/random_cc_type.md - - RandomCCNumber: built_in_transformers/standard_transformers/random_cc_number.md - - RandomCurrency: built_in_transformers/standard_transformers/random_currency.md - - RandomAmountWithCurrency: built_in_transformers/standard_transformers/random_amount_with_currency.md - - RandomName: built_in_transformers/standard_transformers/random_name.md - - RandomLastName: built_in_transformers/standard_transformers/random_last_name.md - - RandomFirstName: built_in_transformers/standard_transformers/random_first_name.md - - RandomFirstNameMale: built_in_transformers/standard_transformers/random_first_name_male.md - - RandomFirstNameFemale: built_in_transformers/standard_transformers/random_first_name_female.md - - RandomTitleMale: built_in_transformers/standard_transformers/random_title_male.md - - RandomTitleFemale: built_in_transformers/standard_transformers/random_title_female.md - - RandomPhoneNumber: built_in_transformers/standard_transformers/random_phone_number.md - - RandomTollFreePhoneNumber: built_in_transformers/standard_transformers/random_toll_free_phone_number.md - - RandomE164PhoneNumber: built_in_transformers/standard_transformers/random_e164_phone_number.md - - RealAddress: built_in_transformers/standard_transformers/real_address.md - - RegexpReplace: built_in_transformers/standard_transformers/regexp_replace.md - - Replace: built_in_transformers/standard_transformers/replace.md - - SetNull: built_in_transformers/standard_transformers/set_null.md - - Advanced transformers: - - built_in_transformers/advanced_transformers/index.md - - Json: built_in_transformers/advanced_transformers/json.md - - Template: built_in_transformers/advanced_transformers/template.md - - TemplateRecord: built_in_transformers/advanced_transformers/template_record.md - - Custom functions: - - built_in_transformers/advanced_transformers/custom_functions/index.md - - Core custom functions: built_in_transformers/advanced_transformers/custom_functions/core_functions.md - - Faker function: built_in_transformers/advanced_transformers/custom_functions/faker_function.md - - Release notes: - - Greenmask 0.1.12: release_notes/greenmask_0_1_12.md - - Greenmask 0.1.11: release_notes/greenmask_0_1_11.md - - Greenmask 0.1.10: release_notes/greenmask_0_1_10.md - - Greenmask 0.1.9: release_notes/greenmask_0_1_9.md - - Greenmask 0.1.8: release_notes/greenmask_0_1_8.md - - Greenmask 0.1.7: release_notes/greenmask_0_1_7.md - - Greenmask 0.1.6: release_notes/greenmask_0_1_6.md - - Greenmask 0.1.5: release_notes/greenmask_0_1_5.md - - Greenmask 0.1.4: release_notes/greenmask_0_1_4.md - - Greenmask 0.1.3: release_notes/greenmask_0_1_3.md - - Greenmask 0.1.2: release_notes/greenmask_0_1_2.md - - Greenmask 0.1.1: release_notes/greenmask_0_1_1.md - - Greenmask 0.1.0: release_notes/greenmask_0_1_0.md - - Greenmask 0.1.0 Beta: release_notes/greenmask_0_1_0_beta.md - -repo_url: https://github.com/GreenmaskIO/greenmask -repo_name: GreenmaskIO/greenmask -site_url: https://greenmask.io/ -copyright: Copyright © 2024 Greenmask - - -extra: - version: - provider: mike - social: - - icon: fontawesome/brands/x-twitter - link: https://twitter.com/GreenmaskIO - - icon: fontawesome/brands/discord - link: https://discord.gg/97AKHdGD - - icon: fontawesome/brands/github - link: https://github.com/GreenmaskIO/greenmask - -plugins: - - social - - search - - mike From 33109375ac073431c87d362258fda8f7209e9378 Mon Sep 17 00:00:00 2001 From: Vladimir Tarbaev Date: Tue, 7 May 2024 09:59:55 +0300 Subject: [PATCH 3/5] setting up multiple versions of docs --- docs/overrides/main.html | 2 +- docs/release_notes/greenmask_0_1_13.md | 12 ++++++++++++ mkdocs.yml | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 docs/release_notes/greenmask_0_1_13.md diff --git a/docs/overrides/main.html b/docs/overrides/main.html index 1635f8f5..0410c73d 100644 --- a/docs/overrides/main.html +++ b/docs/overrides/main.html @@ -1,7 +1,7 @@ {% extends "base.html" %} {% block announce %} - Version 0.1.12 is released + Version 0.1.13 is released {% endblock %} {% block outdated %} diff --git a/docs/release_notes/greenmask_0_1_13.md b/docs/release_notes/greenmask_0_1_13.md new file mode 100644 index 00000000..bfee60b9 --- /dev/null +++ b/docs/release_notes/greenmask_0_1_13.md @@ -0,0 +1,12 @@ +# Greenmask 0.1.13 + +This release introduces only improvements in documentation deployment. The core greenmask utility does not contain any changes. + +## Changes + +* Added documentation deployment with versioning + +## Assets + +To download the Greenmask binary compatible with your system, see +the [release's assets list](https://github.com/GreenmaskIO/greenmask/releases/tag/v0.1.13). \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index a99fe076..336a2d99 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -117,6 +117,7 @@ nav: - Core custom functions: built_in_transformers/advanced_transformers/custom_functions/core_functions.md - Faker function: built_in_transformers/advanced_transformers/custom_functions/faker_function.md - Release notes: + - Greenmask 0.1.13: release_notes/greenmask_0_1_13.md - Greenmask 0.1.12: release_notes/greenmask_0_1_12.md - Greenmask 0.1.11: release_notes/greenmask_0_1_11.md - Greenmask 0.1.10: release_notes/greenmask_0_1_10.md From 3d4c63b3784b76a7d4a68452ae0669ac8cca5bca Mon Sep 17 00:00:00 2001 From: Vadim Voitenko Date: Tue, 7 May 2024 22:33:27 +0300 Subject: [PATCH 4/5] fix: fixed large object dumper panic caused by wrong function enclosure Closes #102 --- internal/db/postgres/dumpers/large_object.go | 125 ++++++++++--------- 1 file changed, 66 insertions(+), 59 deletions(-) diff --git a/internal/db/postgres/dumpers/large_object.go b/internal/db/postgres/dumpers/large_object.go index edaa8f80..cfd5c86f 100644 --- a/internal/db/postgres/dumpers/large_object.go +++ b/internal/db/postgres/dumpers/large_object.go @@ -56,67 +56,10 @@ func (lod *BlobsDumper) Execute(ctx context.Context, tx pgx.Tx, st storages.Stor w, r := countwriter.NewGzipPipe() // Writing goroutine - eg.Go( - func() error { - defer func() { - log.Debug(). - Uint32("oid", uint32(lo.Oid)). - Msg("closing reader") - if err := r.Close(); err != nil { - log.Warn(). - Err(err). - Uint32("oid", uint32(lo.Oid)). - Msg("error closing LargeObject reader") - } - }() - err := st.PutObject(gtx, fmt.Sprintf("blob_%d.dat.gz", lo.Oid), r) - if err != nil { - return fmt.Errorf("cannot write large object %d object: %w", lo.Oid, err) - } - return nil - }, - ) + eg.Go(largeObjectWriter(gtx, st, lo, r)) // Dumping goroutine - eg.Go( - func() error { - defer func() { - log.Debug(). - Uint32("oid", uint32(lo.Oid)). - Msg("closing writer") - if err := w.Close(); err != nil { - log.Warn().Err(err).Msg("error closing Blobs writer") - } - }() - buf := make([]byte, loBufSize) - los := tx.LargeObjects() - lo, err := los.Open(ctx, uint32(lo.Oid), pgx.LargeObjectModeRead) - defer func() { - if err := lo.Close(); err != nil { - log.Warn().Err(err).Msg("error closing large object") - } - }() - if err != nil { - return fmt.Errorf("error dumping large object %d: %w", lo, err) - } - var done bool - for !done { - size, err := lo.Read(buf) - if err != nil { - if errors.Is(err, io.EOF) { - buf = buf[:size] - done = true - } else { - return fmt.Errorf("error reading large object %d: %w", lo, err) - } - } - if _, err = w.Write(buf); err != nil { - return fmt.Errorf("error writing large object %d into storage: %w", lo, err) - } - } - return nil - }, - ) + eg.Go(largeObjectDumper(gtx, lo, w, tx)) if err := eg.Wait(); err != nil { return nil, err @@ -145,6 +88,70 @@ func (lod *BlobsDumper) Execute(ctx context.Context, tx pgx.Tx, st storages.Stor return lod.Blobs, nil } +func largeObjectWriter(ctx context.Context, st storages.Storager, lo *entries.LargeObject, r countwriter.CountReadCloser) func() error { + return func() error { + defer func() { + log.Debug(). + Uint32("oid", uint32(lo.Oid)). + Msg("closing reader") + if err := r.Close(); err != nil { + log.Warn(). + Err(err). + Uint32("oid", uint32(lo.Oid)). + Msg("error closing LargeObject reader") + } + }() + err := st.PutObject(ctx, fmt.Sprintf("blob_%d.dat.gz", lo.Oid), r) + if err != nil { + return fmt.Errorf("cannot write large object %d object: %w", lo.Oid, err) + } + return nil + } +} + +func largeObjectDumper(ctx context.Context, lo *entries.LargeObject, w countwriter.CountWriteCloser, tx pgx.Tx) func() error { + return func() error { + defer func() { + log.Debug(). + Uint32("oid", uint32(lo.Oid)). + Msg("closing writer") + if err := w.Close(); err != nil { + log.Warn().Err(err).Msg("error closing Blobs writer") + } + }() + buf := make([]byte, loBufSize) + los := tx.LargeObjects() + loObj, err := los.Open(ctx, uint32(lo.Oid), pgx.LargeObjectModeRead) + if err != nil { + return fmt.Errorf("error opening large object %d: %w", lo.Oid, err) + } + defer func() { + if err := loObj.Close(); err != nil { + log.Warn().Err(err).Msg("error closing large object") + } + }() + if err != nil { + return fmt.Errorf("error dumping large object %d: %w", lo.Oid, err) + } + var done bool + for !done { + size, err := loObj.Read(buf) + if err != nil { + if errors.Is(err, io.EOF) { + buf = buf[:size] + done = true + } else { + return fmt.Errorf("error reading large object %d: %w", lo.Oid, err) + } + } + if _, err = w.Write(buf); err != nil { + return fmt.Errorf("error writing large object %d into storage: %w", lo.Oid, err) + } + } + return nil + } +} + func (lod *BlobsDumper) DebugInfo() string { var largeObjects []string for _, lo := range lod.Blobs.LargeObjects { From e731cccaa103502a12872f5e16ce13e926abe76d Mon Sep 17 00:00:00 2001 From: Vadim Voitenko Date: Wed, 8 May 2024 12:15:41 +0300 Subject: [PATCH 5/5] fix: refactored logic for blobs.toc writing Closes #102 --- internal/db/postgres/dumpers/large_object.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/internal/db/postgres/dumpers/large_object.go b/internal/db/postgres/dumpers/large_object.go index cfd5c86f..6dcdfc9b 100644 --- a/internal/db/postgres/dumpers/large_object.go +++ b/internal/db/postgres/dumpers/large_object.go @@ -73,6 +73,16 @@ func (lod *BlobsDumper) Execute(ctx context.Context, tx pgx.Tx, st storages.Stor Msg("dumping large object completed") } + // Writing blobs.toc + if err := lod.generateBlobsToc(ctx, st); err != nil { + return nil, fmt.Errorf("cannot write large object blobs.toc: %w", err) + } + + return lod.Blobs, nil +} + +func (lod *BlobsDumper) generateBlobsToc(ctx context.Context, st storages.Storager) error { + log.Debug().Msg("writing blobs.toc") // Writing blobs.toc blobsTocBuf := bytes.NewBuffer(nil) @@ -82,10 +92,9 @@ func (lod *BlobsDumper) Execute(ctx context.Context, tx pgx.Tx, st storages.Stor err := st.PutObject(ctx, "blobs.toc", blobsTocBuf) if err != nil { - return nil, fmt.Errorf("cannot write large object blobs.toc: %w", err) + return fmt.Errorf("cannot write large object blobs.toc: %w", err) } - - return lod.Blobs, nil + return nil } func largeObjectWriter(ctx context.Context, st storages.Storager, lo *entries.LargeObject, r countwriter.CountReadCloser) func() error {